From 4c6e609979802c26228ced88f7d158231706eea8 Mon Sep 17 00:00:00 2001 From: Revar Desmera Date: Mon, 26 Oct 2026 08:55:28 -0700 Subject: [PATCH] Lexer: U+2028, U+2029 and U+0085 are whitespace that ends a // comment Qt's editor types U+2028 LINE SEPARATOR for Shift+Enter and shows it as a new line, but the lexer rejected it as an unexpected character (BelfrySCAD #949). The three Unicode line separators are now whitespace, and a line comment ends at one, so what follows it on screen is code. They do not advance the line counter: BelfrySCAD's editor numbers lines by block and a U+2028 sits inside one, so lines and columns after it still match the editor's. Byte offsets are untouched either way. Co-Authored-By: Claude Opus 5.5 --- src/grammar/lexer.l | 12 +++++++++++- tests/test_lexical.cpp | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/grammar/lexer.l b/src/grammar/lexer.l index 3817af5..8c0a66a 100644 --- a/src/grammar/lexer.l +++ b/src/grammar/lexer.l @@ -88,7 +88,17 @@ NUMLIT 0[xX][0-9A-Fa-f]+|[0-9]+([.][0-9]*)?([eE][+-]?[0-9]+)?|[.][0-9]+([eE][+- // U+FEFF (EF BB BF, a byte-order mark, anywhere in the file) and a bare // A0, Latin-1's no-break space. } -"//"[^\n]* { /* skip line comment */ } +\xe2\x80\xa8|\xe2\x80\xa9|\xc2\x85 { + // U+2028 LINE SEPARATOR, U+2029 PARAGRAPH SEPARATOR and U+0085 NEXT + // LINE: line breaks to look at, so whitespace that also ends a `//` + // comment (below). Qt's editor types U+2028 for Shift+Enter, and it + // shows as a new line (BelfrySCAD #949). It does NOT count as a line: + // the editor numbers its lines by block and a U+2028 sits inside one, + // so every line and column after it still matches the editor's. + // OpenSCAD's lexer rejects all three. +} + /* A line comment ends at a newline or at any separator above. */ +"//"([^\n\xe2\xc2]|\xe2[^\x80\n]|\xe2\x80[^\xa8\xa9\n]|\xc2[^\x85\n])* { /* skip line comment */ } "/*"([^*]|\*+[^*/])*\*+"/" { /* skip block comment */ } {IDENT} { diff --git a/tests/test_lexical.cpp b/tests/test_lexical.cpp index fdf7696..8d579ff 100644 --- a/tests/test_lexical.cpp +++ b/tests/test_lexical.cpp @@ -483,6 +483,22 @@ TEST(LexicalWhitespace, NoBreakSpaceByteOrderMarkAndLatin1A0) { } } +// U+2028/U+2029/U+0085 are whitespace that ends a line comment, without +// counting as a line (BelfrySCAD's editor numbers lines by block, and a +// U+2028 sits inside one). +TEST(LexicalWhitespace, UnicodeLineSeparatorsEndACommentButNotALine) { + for (const char* sep : {"\xe2\x80\xa8", "\xe2\x80\xa9", "\xc2\x85"}) { + const std::string src = std::string("a = 1;") + sep + "b = 2; // note" + sep + "c = 3;\nd = 4;"; + auto ast = parseSrc(src); + ASSERT_EQ(ast.size(), 4u) << "the comment swallowed c = 3"; + EXPECT_EQ(ast[2]->position().line, 1); + EXPECT_EQ(size_t(ast[2]->position().start_offset), src.find("c = 3")); + EXPECT_EQ(ast[3]->position().line, 2); + } + // Other multi-byte characters stay inside the comment. + EXPECT_EQ(parseSrc("a = 1; // \xe2\x80\x94 \xc2\xa0 \xe2\x82\xac c = 3;\nb = 2;").size(), 2u); +} + // The message quotes a whole character, and names a byte that starts none, // so it is always valid UTF-8. TEST(LexicalWhitespace, UnexpectedCharacterMessageIsValidUtf8) {