Validate field size after reading the whole record

doc/csv/options/parsing/field_size_limit.rdoc

@@ -1,18 +1,14 @@
 ====== Option +field_size_limit+
 
 Specifies the \Integer field size limit.
+This is the maximum field size + 1 allowed.
+
+Deprecated since 3.2.3. Use {max_field_size}[#class-CSV-label-Option+max_field_size] instead.
+<tt>field_size_limit: n</tt> is the same as <tt>max_field_size: n - 1</tt>.
 
 Default value:
   CSV::DEFAULT_OPTIONS.fetch(:field_size_limit) # => nil
 
-This is a maximum size CSV will read ahead looking for the closing quote for a field.
-(In truth, it reads to the first line ending beyond this size.)
-If a quote cannot be found within the limit CSV will raise a MalformedCSVError,
-assuming the data is faulty.
-You can use this limit to prevent what are effectively DoS attacks on the parser.
-However, this limit can cause a legitimate parse to fail;
-therefore the default value is +nil+ (no limit).
-
 For the examples in this section:
   str = <<~EOT
     "a","b"
@@ -26,14 +22,12 @@ Using the default +nil+:
   ary = CSV.parse(str)
   ary # => [["a", "b"], ["\n2345\n", ""]]
 
-Using <tt>50</tt>:
-  field_size_limit = 50
-  ary = CSV.parse(str, field_size_limit: field_size_limit)
+Using <tt>7</tt>:
+  ary = CSV.parse(str, field_size_limit: 7)
   ary # => [["a", "b"], ["\n2345\n", ""]]
 
 ---
 
 Raises an exception if a field is too long:
-  big_str = "123456789\n" * 1024
-  # Raises CSV::MalformedCSVError (Field size exceeded in line 1.)
-  CSV.parse('valid,fields,"' + big_str + '"', field_size_limit: 2048)
+  # Raises CSV::MalformedCSVError (Field size exceeded: 6 > 5 in line 2.)
+  CSV.parse(str, field_size_limit: 6)

doc/csv/options/parsing/max_field_size.rdoc

@@ -0,0 +1,50 @@
+====== Option +max_field_size+
+
+Specifies the \Integer maximum field size in characters.
+
+Default value:
+  CSV::DEFAULT_OPTIONS.fetch(:max_field_size) # => nil
+
+If a field is longer than this size,
+CSV raises a MalformedCSVError for the record.
+The size is checked after the whole record is read.
+You can continue to read the next record after the error;
+the record that has the long field is skipped.
+
+This doesn't limit how much data CSV reads.
+For example, CSV reads the whole record that has a long field,
+and an unclosed quoted field is read to the end of the input.
+If you need to limit memory usage, limit the input size yourself.
+
+For examples in this section:
+  str = <<~EOT
+    a,b
+    abcde,"x
+    y"
+    c,d
+  EOT
+  str # => "a,b\nabcde,\"x\ny\"\nc,d\n"
+
+Using the default +nil+:
+  ary = CSV.parse(str)
+  ary # => [["a", "b"], ["abcde", "x\ny"], ["c", "d"]]
+
+Using <tt>5</tt>:
+  ary = CSV.parse(str, max_field_size: 5)
+  ary # => [["a", "b"], ["abcde", "x\ny"], ["c", "d"]]
+
+---
+
+Raises an exception if a field is too long:
+  # Raises CSV::MalformedCSVError (Field size exceeded: 5 > 4 in line 2.)
+  CSV.parse(str, max_field_size: 4)
+
+Continues to read the next record after the exception:
+  csv = CSV.new(str, max_field_size: 4)
+  csv.shift # => ["a", "b"]
+  begin
+    csv.shift
+  rescue CSV::MalformedCSVError
+  end
+  csv.line # => "abcde,\"x\ny\"\n"
+  csv.shift # => ["c", "d"]

lib/csv.rb

@@ -374,6 +374,8 @@
 #
 # :include: ../doc/csv/options/parsing/field_size_limit.rdoc
 #
+# :include: ../doc/csv/options/parsing/max_field_size.rdoc
+#
 # :include: ../doc/csv/options/parsing/converters.rdoc
 #
 # :include: ../doc/csv/options/parsing/unconverted_fields.rdoc

lib/csv/parser.rb

@@ -942,7 +942,6 @@ def skip_line?(line)
     def validate_field_size(field)
       return unless @max_field_size
       return if field.size <= @max_field_size
-      ignore_broken_line
       message = "Field size exceeded: #{field.size} > #{@max_field_size}"
       raise MalformedCSVError.new(message, @lineno)
     end
@@ -959,11 +958,6 @@ def parse_no_quote(&block)
         else
           line = strip_value(line)
           row = line.split(@split_column_separator, -1)
-          if @max_field_size
-            row.each do |column|
-              validate_field_size(column)
-            end
-          end
           n_columns = row.size
           i = 0
           while i < n_columns
@@ -1028,7 +1022,6 @@ def parse_quotable_loose(&block)
                   @parse_method = :parse_quotable_robust
                   return parse_quotable_robust(&block)
                 end
-                validate_field_size(row[i])
               end
               i += 1
             end
@@ -1055,7 +1048,6 @@ def parse_quotable_robust(&block)
         value = parse_column_value
         if value
           @scanner.scan_all(@strip_value) if @strip_value
-          validate_field_size(value)
         end
         if parse_column_end
           row << value
@@ -1276,6 +1268,12 @@ def start_row
     def emit_row(row, quoted_fields=FieldsConverter::NO_QUOTED_FIELDS, &block)
       @lineno += 1
 
+      if @max_field_size
+        row.each do |field|
+          validate_field_size(field) if field
+        end
+      end
+
       raw_row = row
       if @use_headers
         if @headers.nil?

test/csv/parse/test_general.rb

@@ -329,6 +329,70 @@ def test_max_field_size_in_extended_column_exceeding
     assert_parse_errors_out(data, max_field_size: 4)
   end
 
+  def test_max_field_size_recover_no_quote
+    assert_equal([
+                   "Field size exceeded: 5 > 4 in line 1.",
+                   "abcde,b\n",
+                   ["c", "d"],
+                   2,
+                 ],
+                 parse_after_error("abcde,b\nc,d\n",
+                                   max_field_size: 4,
+                                   quote_char: nil))
+  end
+
+  def test_max_field_size_recover_quotable_loose
+    assert_equal([
+                   "Field size exceeded: 5 > 4 in line 1.",
+                   "abcde,b\n",
+                   ["c", "d"],
+                   2,
+                 ],
+                 parse_after_error("abcde,b\nc,d\n", max_field_size: 4))
+  end
+
+  def test_max_field_size_recover_multiline_record
+    assert_equal([
+                   "Field size exceeded: 5 > 4 in line 1.",
+                   %Q{abcde,"x\ny"\n},
+                   ["c", "d"],
+                   2,
+                 ],
+                 parse_after_error(%Q{abcde,"x\ny"\nc,d\n}, max_field_size: 4))
+  end
+
+  def test_max_field_size_recover_multiline_record_strip
+    assert_equal([
+                   "Field size exceeded: 5 > 4 in line 1.",
+                   %Q{abcde , "x\ny" \n},
+                   ["c", "d"],
+                   2,
+                 ],
+                 parse_after_error(%Q{abcde , "x\ny" \nc,d\n},
+                                   max_field_size: 4,
+                                   strip: true))
+  end
+
+  def test_max_field_size_malformed_record
+    assert_equal([
+                   "Any value after quoted field isn't allowed in line 1.",
+                   %Q{"abcde"junk,b\n},
+                   ["c", "d"],
+                   2,
+                 ],
+                 parse_after_error(%Q{"abcde"junk,b\nc,d\n}, max_field_size: 4))
+  end
+
+  def test_max_field_size_unclosed_quote
+    assert_equal([
+                   "Unclosed quoted field in line 1.",
+                   %Q{abcde,"x\nc,d\n},
+                   nil,
+                   1,
+                 ],
+                 parse_after_error(%Q{abcde,"x\nc,d\n}, max_field_size: 4))
+  end
+
   def test_row_sep_auto_cr
     assert_equal([["a"]], CSV.parse("a\r"))
   end
@@ -383,4 +447,12 @@ def assert_parse_errors_out(data, timeout: PARSE_ERROR_TIMEOUT, **options)
       end
     end
   end
+
+  def parse_after_error(data, **options)
+    csv = CSV.new(data, **options)
+    error = assert_raise(CSV::MalformedCSVError) do
+      csv.shift
+    end
+    [error.message, csv.line, csv.shift, csv.lineno]
+  end
 end