JSON: Add opt-in comments and trailing comma flags

UPGRADING

@@ -44,6 +44,16 @@ PHP 8.7 UPGRADE NOTES
     A ValueError is thrown if the array is empty or contains an invalid
     locale, and a TypeError if it contains a non-string value.
 
+- JSON:
+  . json_decode() and json_validate() can now accept comments and trailing
+    commas. JSON_ALLOW_COMMENTS accepts "//" line comments and "/* */"
+    block comments wherever whitespace is allowed. Block comments do not
+    nest. A line comment ends at LF, CR, CRLF, or the end of the input.
+    A NUL byte in a comment is a control character error.
+    JSON_ALLOW_TRAILING_COMMAS accepts one comma after the last element of
+    a non-empty array or the last member of a non-empty object. Without
+    these flags, parsing does not change.
+
 - Streams:
   . The so_linger, so_rcvbuf and so_sndbuf stream socket context options are
     now also applied to unix and unix datagram sockets.
@@ -60,6 +70,13 @@ PHP 8.7 UPGRADE NOTES
 5. Changed Functions
 ========================================
 
+- JSON:
+  . json_decode() now accepts the JSON_ALLOW_COMMENTS and
+    JSON_ALLOW_TRAILING_COMMAS flags.
+  . json_validate() now accepts the JSON_ALLOW_COMMENTS and
+    JSON_ALLOW_TRAILING_COMMAS flags, in addition to
+    JSON_INVALID_UTF8_IGNORE.
+
 ========================================
 6. New Functions
 ========================================
@@ -80,6 +97,10 @@ PHP 8.7 UPGRADE NOTES
 10. New Global Constants
 ========================================
 
+- JSON:
+  . JSON_ALLOW_COMMENTS.
+  . JSON_ALLOW_TRAILING_COMMAS.
+
 ========================================
 11. Changes to INI File Handling
 ========================================

ext/json/json.c

@@ -350,8 +350,11 @@ PHP_FUNCTION(json_validate)
 	ZEND_PARSE_PARAMETERS_END();
 
 
-	if ((options != 0) && (options != PHP_JSON_INVALID_UTF8_IGNORE)) {
-		zend_argument_value_error(3, "must be a valid flag (allowed flags: JSON_INVALID_UTF8_IGNORE)");
+	if ((options & ~(PHP_JSON_INVALID_UTF8_IGNORE
+			| PHP_JSON_ALLOW_COMMENTS
+			| PHP_JSON_ALLOW_TRAILING_COMMAS)) != 0) {
+		zend_argument_value_error(3, "must be a valid flag (allowed flags: "
+			"JSON_INVALID_UTF8_IGNORE, JSON_ALLOW_COMMENTS, JSON_ALLOW_TRAILING_COMMAS)");
 		RETURN_THROWS();
 	}
 

ext/json/json.stub.php

@@ -90,6 +90,17 @@
  */
 const JSON_THROW_ON_ERROR = UNKNOWN;
 
+/**
+ * @var int
+ * @cvalue PHP_JSON_ALLOW_COMMENTS
+ */
+const JSON_ALLOW_COMMENTS = UNKNOWN;
+/**
+ * @var int
+ * @cvalue PHP_JSON_ALLOW_TRAILING_COMMAS
+ */
+const JSON_ALLOW_TRAILING_COMMAS = UNKNOWN;
+
 /**
  * @var int
  * @cvalue PHP_JSON_ERROR_NONE

ext/json/json_parser.y

@@ -119,6 +119,7 @@ members:
 				}
 			}
 	|	member
+	|	member trailing_comma
 ;
 
 member:
@@ -175,6 +176,7 @@ elements:
 				}
 			}
 	|	element
+	|	element trailing_comma
 ;
 
 element:
@@ -190,6 +192,18 @@ element:
 			}
 ;
 
+trailing_comma:
+		','
+			{
+				if (!(parser->scanner.options & PHP_JSON_ALLOW_TRAILING_COMMAS)) {
+					/* This rule also reduces on an error token from the scanner,
+					 * so keep the scanner's error code. */
+					php_json_yyerror(parser, NULL);
+					YYERROR;
+				}
+			}
+;
+
 key:
 		PHP_JSON_T_STRING
 	|	PHP_JSON_T_ESTRING

ext/json/json_scanner.re

@@ -102,6 +102,11 @@ void php_json_scanner_init(php_json_scanner *s, const char *str, size_t str_len,
 
 int php_json_scan(php_json_scanner *s)
 {
+	/* Line of the last block comment opener, for an unterminated comment.
+	 * s->token keeps the opener position while in COMMENT_BLOCK. */
+	php_json_ctype *comment_line_start = s->line_start;
+	uint64_t comment_line = s->line;
+
 	ZVAL_NULL(&s->value);
 
 std:
@@ -203,6 +208,22 @@ std:
 		goto std;
 	}
 	<JS>WS                   { goto std; }
+	<JS>"//"                 {
+		if (!(s->options & PHP_JSON_ALLOW_COMMENTS)) {
+			s->errcode = PHP_JSON_ERROR_SYNTAX;
+			return PHP_JSON_T_ERROR;
+		}
+		PHP_JSON_CONDITION_SET_AND_GOTO(COMMENT_LINE);
+	}
+	<JS>"/*"                 {
+		if (!(s->options & PHP_JSON_ALLOW_COMMENTS)) {
+			s->errcode = PHP_JSON_ERROR_SYNTAX;
+			return PHP_JSON_T_ERROR;
+		}
+		comment_line_start = s->line_start;
+		comment_line = s->line;
+		PHP_JSON_CONDITION_SET_AND_GOTO(COMMENT_BLOCK);
+	}
 	<JS>EOI                  {
 		if (s->limit < s->cursor) {
 			return PHP_JSON_T_EOI;
@@ -230,6 +251,65 @@ std:
 		s->errcode = PHP_JSON_ERROR_UTF8;
 		return PHP_JSON_T_ERROR;
 	}
+	<COMMENT_LINE>NL         {
+		s->line++;
+		s->line_start = s->cursor;
+		PHP_JSON_CONDITION_SET(JS);
+		goto std;
+	}
+	<COMMENT_LINE>"\r"       {
+		/* A bare CR ends the comment but is whitespace, as in the JS condition. */
+		PHP_JSON_CONDITION_SET(JS);
+		goto std;
+	}
+	<COMMENT_LINE>EOI        {
+		if (s->limit < s->cursor) {
+			s->token = s->limit;
+			PHP_JSON_CONDITION_SET(JS);
+			return PHP_JSON_T_EOI;
+		}
+		s->token = s->cursor - 1;
+		s->errcode = PHP_JSON_ERROR_CTRL_CHAR;
+		return PHP_JSON_T_ERROR;
+	}
+	<COMMENT_LINE>UTF8       { PHP_JSON_CONDITION_GOTO(COMMENT_LINE); }
+	<COMMENT_LINE>ANY        {
+		if (s->options & (PHP_JSON_INVALID_UTF8_IGNORE | PHP_JSON_INVALID_UTF8_SUBSTITUTE)) {
+			PHP_JSON_CONDITION_GOTO(COMMENT_LINE);
+		}
+		s->token = s->cursor - 1;
+		s->errcode = PHP_JSON_ERROR_UTF8;
+		return PHP_JSON_T_ERROR;
+	}
+	<COMMENT_BLOCK>"*/"      {
+		PHP_JSON_CONDITION_SET(JS);
+		goto std;
+	}
+	<COMMENT_BLOCK>NL        {
+		s->line++;
+		s->line_start = s->cursor;
+		PHP_JSON_CONDITION_GOTO(COMMENT_BLOCK);
+	}
+	<COMMENT_BLOCK>EOI       {
+		if (s->limit < s->cursor) {
+			s->line_start = comment_line_start;
+			s->line = comment_line;
+			s->errcode = PHP_JSON_ERROR_SYNTAX;
+			return PHP_JSON_T_ERROR;
+		}
+		s->token = s->cursor - 1;
+		s->errcode = PHP_JSON_ERROR_CTRL_CHAR;
+		return PHP_JSON_T_ERROR;
+	}
+	<COMMENT_BLOCK>UTF8      { PHP_JSON_CONDITION_GOTO(COMMENT_BLOCK); }
+	<COMMENT_BLOCK>ANY       {
+		if (s->options & (PHP_JSON_INVALID_UTF8_IGNORE | PHP_JSON_INVALID_UTF8_SUBSTITUTE)) {
+			PHP_JSON_CONDITION_GOTO(COMMENT_BLOCK);
+		}
+		s->token = s->cursor - 1;
+		s->errcode = PHP_JSON_ERROR_UTF8;
+		return PHP_JSON_T_ERROR;
+	}
 	<STR_P1>EOI             {
 		if (s->limit < s->cursor) {
 			s->errcode = PHP_JSON_ERROR_SYNTAX;

ext/json/php_json.h

@@ -89,6 +89,10 @@ static inline void php_json_error_details_clear(php_json_error_details *out) {
 #define PHP_JSON_INVALID_UTF8_SUBSTITUTE    (1<<21)
 #define PHP_JSON_THROW_ON_ERROR             (1<<22)
 
+/* json_validate() and json_decode() common options */
+#define PHP_JSON_ALLOW_COMMENTS             (1<<23)
+#define PHP_JSON_ALLOW_TRAILING_COMMAS      (1<<24)
+
 /* default depth */
 #define PHP_JSON_PARSER_DEFAULT_DEPTH 512
 

ext/json/tests/json_decode_allow_comments.phpt

@@ -0,0 +1,90 @@
+--TEST--
+json_decode() and json_validate() with JSON_ALLOW_COMMENTS
+--FILE--
+<?php
+
+function decode_and_validate(string $json): void {
+    $value = json_decode($json, true, 512, JSON_ALLOW_COMMENTS);
+    echo json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), "\n";
+    var_dump(json_validate($json, 512, JSON_ALLOW_COMMENTS));
+}
+
+/* Exercise every whitespace position in objects and arrays. */
+decode_and_validate('/* before root */ {
+    /* after object opener */
+    "a" /* after key */ : /* after colon */
+    [ /* after array opener */ 1 /* before comma */,
+      /* after comma */ 2 /* before array closer */ ]
+    /* after value */, /* after member comma */
+    "b"/* before colon */:/* after colon */3/* before object closer */
+} /* after root */');
+
+decode_and_validate('[1 // before comma
+, 2 // before closer
+]');
+decode_and_validate('/* object */{/* only trivia */}');
+decode_and_validate('/* array */[/* only trivia */]');
+
+/* Line comments end at LF, CR, CRLF, or end-of-input. */
+decode_and_validate("// LF\n1");
+decode_and_validate("// CR\r2");
+decode_and_validate("// CRLF\r\n3");
+decode_and_validate("4// EOF");
+decode_and_validate("// U+2028 \u{2028} and U+2029 \u{2029}\n6");
+decode_and_validate("// controls: a\x01\x0cb\n7");
+
+/* Delimiters can touch. The '*' of an opener does not also close the comment. */
+decode_and_validate('/**/1');
+decode_and_validate('/***/1');
+decode_and_validate('/*/ */1');
+
+/* A comment can directly follow a float or an exponent. */
+decode_and_validate('1.5/*x*/');
+decode_and_validate('1e5//x');
+
+/* Comment delimiters inside strings are ordinary string content. */
+decode_and_validate('["//", "/* not a comment */", "https://php.net/a/*b*/"]');
+
+/* Valid Unicode, Unicode line separators, and controls other than NUL are
+ * comment text. */
+decode_and_validate("/* héllo \u{2028} world \u{2029} \x01\x0c */ 5");
+decode_and_validate('/* ' . str_repeat('x', 65536) . ' */ 8');
+
+?>
+--EXPECT--
+{"a":[1,2],"b":3}
+bool(true)
+[1,2]
+bool(true)
+[]
+bool(true)
+[]
+bool(true)
+1
+bool(true)
+2
+bool(true)
+3
+bool(true)
+4
+bool(true)
+6
+bool(true)
+7
+bool(true)
+1
+bool(true)
+1
+bool(true)
+1
+bool(true)
+1.5
+bool(true)
+100000
+bool(true)
+["//","/* not a comment */","https://php.net/a/*b*/"]
+bool(true)
+5
+bool(true)
+8
+bool(true)