From d714f585396069a2d53bb3e70fac3ebd8b2502e2 Mon Sep 17 00:00:00 2001 From: "William T. Nelson" <35801+wtn@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:02:20 -0500 Subject: [PATCH] Skip the whole record after a field size error When parse_quotable_robust found a field longer than max_field_size, it skipped only to the end of the physical line. If a later quoted field in the record spanned lines, the rest of the record was parsed as the next row. Skip the rest of the record with parse_column_value, so the next row and CSV#line are correct. Co-authored-by: Claude Opus 5.5 --- lib/csv/parser.rb | 22 +++++++++- test/csv/parse/test_general.rb | 77 ++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/lib/csv/parser.rb b/lib/csv/parser.rb index 9b8a009b..c0826dc2 100644 --- a/lib/csv/parser.rb +++ b/lib/csv/parser.rb @@ -942,7 +942,11 @@ def skip_line?(line) def validate_field_size(field) return unless @max_field_size return if field.size <= @max_field_size - ignore_broken_line + if block_given? + yield + else + ignore_broken_line + end message = "Field size exceeded: #{field.size} > #{@max_field_size}" raise MalformedCSVError.new(message, @lineno) end @@ -1055,7 +1059,7 @@ def parse_quotable_robust(&block) value = parse_column_value if value @scanner.scan_all(@strip_value) if @strip_value - validate_field_size(value) + validate_field_size(value) {ignore_broken_record} end if parse_column_end row << value @@ -1264,6 +1268,20 @@ def ignore_broken_line @lineno += 1 end + def ignore_broken_record + lineno = @lineno + begin + while parse_column_end + @scanner.scan_all(@strip_value) if @strip_value + parse_column_value + @scanner.scan_all(@strip_value) if @strip_value + end + ignore_broken_line unless parse_row_end + rescue MalformedCSVError + end + @lineno = lineno + 1 + end + def start_row if @last_line @last_line = nil diff --git a/test/csv/parse/test_general.rb b/test/csv/parse/test_general.rb index 520eb8f0..ebad90b2 100644 --- a/test/csv/parse/test_general.rb +++ b/test/csv/parse/test_general.rb @@ -329,6 +329,83 @@ def test_max_field_size_in_extended_column_exceeding assert_parse_errors_out(data, max_field_size: 4) end + def test_max_field_size_multiline_record_liberal_parsing + csv = CSV.new(%Q{abcde,"x\ny"\nc,d\n}, + max_field_size: 4, liberal_parsing: true) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{abcde,"x\ny"\n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(2, csv.lineno) + end + + def test_max_field_size_three_line_record_liberal_parsing + csv = CSV.new(%Q{abcde,"x\ny\nz"\nc,d\n}, + max_field_size: 4, liberal_parsing: true) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{abcde,"x\ny\nz"\n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(2, csv.lineno) + end + + def test_max_field_size_multiline_record_after_multiline_row + csv = CSV.new(%Q{"a\nb",c\nabcde,"x\ny"\nc,d\n}, max_field_size: 4) + assert_equal(["a\nb", "c"], csv.shift) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 2.", + error.message) + assert_equal(%Q{abcde,"x\ny"\n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(3, csv.lineno) + end + + def test_max_field_size_multiline_record_strip + csv = CSV.new(%Q{abcde , "x\ny" \nc,d\n}, + max_field_size: 4, strip: true) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{abcde , "x\ny" \n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(2, csv.lineno) + end + + def test_max_field_size_value_after_quoted_field + csv = CSV.new(%Q{"abcde"junk,b\nc,d\n}, max_field_size: 4) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{"abcde"junk,b\n}, csv.line) + assert_equal(["c", "d"], csv.shift) + assert_equal(2, csv.lineno) + end + + def test_max_field_size_unclosed_quote_liberal_parsing + csv = CSV.new(%Q{abcde,"x\nc,d\n}, + max_field_size: 4, liberal_parsing: true) + error = assert_raise(CSV::MalformedCSVError) do + csv.shift + end + assert_equal("Field size exceeded: 5 > 4 in line 1.", + error.message) + assert_equal(%Q{abcde,"x\nc,d\n}, csv.line) + assert_nil(csv.shift) + assert_equal(1, csv.lineno) + end + def test_row_sep_auto_cr assert_equal([["a"]], CSV.parse("a\r")) end