From 94ba3b6796c6dd6732774c21fc482ffc2987677b Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 27 Oct 2026 18:39:15 -0600 Subject: [PATCH 1/3] fix(schema_engine): follow ECMA-262 semantics in pattern JSON Schema defines `pattern` and `patternProperties` as ECMA-262 regular expressions. Julia compiles PCRE with UCP and without DOLLAR_ENDONLY, so `$` matched before a trailing newline and `\d`, `\w` matched Unicode digits and letters. - Compile with DOLLAR_ENDONLY and without UCP: `$` matches only at the end, and `\d`, `\w`, `\b` are ASCII. - Translate `\s` and `\S` to ECMA-262 white space: tab, vertical tab, form feed, every Zs, U+FEFF, and the line terminators. - Map Unicode's extra General_Category aliases (digit, cntrl, punct, Combining_Mark). - Run optional/ecmascript-regex.json from the official suite for each draft, and add unit tests that run by default. --- src/schema_engine/dialects.jl | 40 ++++++++++++++++++++++++++++++++-- test/schema_engine/compiled.jl | 18 +++++++++++++++ test/schema_engine/official.jl | 17 +++++++++++++-- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/schema_engine/dialects.jl b/src/schema_engine/dialects.jl index 85f1b0f..736b843 100644 --- a/src/schema_engine/dialects.jl +++ b/src/schema_engine/dialects.jl @@ -225,8 +225,14 @@ end dialect(::Bool; default::Dialect = DRAFT7) = default +# Long names and Unicode's extra aliases (`digit`, `cntrl`, `punct`, `Combining_Mark`) of +# the General_Category values. PCRE knows only the short names. const ECMA_GENERAL_CATEGORIES = Dict( "Cased_Letter" => "L&", + "Combining_Mark" => "M", + "cntrl" => "Cc", + "digit" => "Nd", + "punct" => "P", "Close_Punctuation" => "Pe", "Connector_Punctuation" => "Pc", "Control" => "Cc", @@ -282,12 +288,28 @@ function _ecma_property(property::AbstractString) return String(property) end +# ECMA-262 `\s`: WhiteSpace (tab, vertical tab, form feed, every Zs, U+FEFF) and +# LineTerminator (LF, CR, U+2028, U+2029). PCRE without UCP matches only the ASCII ones. +# Julia compiles with ALT_BSUX, so code points use the JavaScript `\\uhhhh` form. +const ECMA_WHITESPACE = "\\t\\n\\x0B\\f\\r\\p{Zs}\\u2028\\u2029\\uFEFF" + +function _ecma_whitespace(escaped::Char, in_class::Bool) + if in_class + # PCRE cannot subtract a set inside a class, so `[\S]` keeps its ASCII meaning. + return escaped == 's' ? ECMA_WHITESPACE : "\\S" + end + return escaped == 's' ? "[$ECMA_WHITESPACE]" : "[^$ECMA_WHITESPACE]" +end + function _ecma_pattern(pattern::AbstractString) io = IOBuffer() + in_class = false i = firstindex(pattern) stop = lastindex(pattern) while i <= stop if pattern[i] != '\\' + pattern[i] == '[' && (in_class = true) + pattern[i] == ']' && (in_class = false) write(io, pattern[i]) i = nextind(pattern, i) continue @@ -301,7 +323,15 @@ function _ecma_pattern(pattern::AbstractString) write(io, '\\') end isodd(slashes) || continue - i <= stop && pattern[i] in ('p', 'P') || continue + i <= stop || continue + if pattern[i] in ('s', 'S') + seek(io, position(io) - 1) + truncate(io, position(io)) + write(io, _ecma_whitespace(pattern[i], in_class)) + i = nextind(pattern, i) + continue + end + pattern[i] in ('p', 'P') || continue property_type = pattern[i] brace = nextind(pattern, i) brace <= stop && pattern[brace] == '{' || continue @@ -320,4 +350,10 @@ function _ecma_pattern(pattern::AbstractString) return String(take!(io)) end -_ecma_regex(pattern::AbstractString) = Regex(_ecma_pattern(pattern)) +# ECMA-262 semantics on PCRE: `$` matches only at the very end, never before a trailing +# newline (DOLLAR_ENDONLY), and `\d`, `\w`, `\b` are ASCII (no UCP). `\p{...}` still +# matches Unicode properties, which needs only UTF mode. +const ECMA_COMPILE_OPTIONS = (Base.DEFAULT_COMPILER_OPTS & ~Base.PCRE.UCP) | Base.PCRE.DOLLAR_ENDONLY + +_ecma_regex(pattern::AbstractString) = + Regex(_ecma_pattern(pattern), ECMA_COMPILE_OPTIONS, Base.DEFAULT_MATCH_OPTS) diff --git a/test/schema_engine/compiled.jl b/test/schema_engine/compiled.jl index c3011f8..5d671b3 100644 --- a/test/schema_engine/compiled.jl +++ b/test/schema_engine/compiled.jl @@ -237,6 +237,24 @@ end Dict("pattern" => "["); dialect = SchemaEngine.DRAFT202012, ) + + # ECMA-262 `pattern` semantics, which PCRE's defaults do not follow. The official + # suite's optional/ecmascript-regex.json covers this fully (OPENAPI_SCHEMA_SUITE=all). + ecma(pattern) = SchemaEngine.CompiledSchema( + Dict("pattern" => pattern); + dialect = SchemaEngine.DRAFT202012, + ) + @test !isvalid(ecma("^abc\$"), "abc\n") + @test isvalid(ecma("^\\d\$"), "7") + @test !isvalid(ecma("^\\d\$"), "\u07c0") + @test !isvalid(ecma("^\\w\$"), "é") + @test isvalid(ecma("^\\s\$"), "\u00a0") + @test isvalid(ecma("^\\s\$"), "\ufeff") + @test isvalid(ecma("^\\s\$"), "\u2029") + @test !isvalid(ecma("^\\S\$"), "\u2029") + @test isvalid(ecma("^[\\s]\$"), "\u3000") + @test isvalid(ecma("^\\p{digit}+\$"), "\u07c0\u0663") + @test isvalid(ecma("^\\p{Letter}cole\$"), "école") @test_throws SchemaEngine.CompilationError SchemaEngine.CompiledSchema( Dict("contains" => Dict(), "minContains" => "invalid"); dialect = SchemaEngine.DRAFT202012, diff --git a/test/schema_engine/official.jl b/test/schema_engine/official.jl index 1601945..ea44d1b 100644 --- a/test/schema_engine/official.jl +++ b/test/schema_engine/official.jl @@ -34,8 +34,13 @@ function _official_schema_suite_dir() ) end -function _test_compiled_draft(directory, retriever, schema_dialect, base_uri) - files = sort(filter(name -> endswith(name, ".json"), readdir(directory))) +function _test_compiled_draft( + directory, + retriever, + schema_dialect, + base_uri; + files = sort(filter(name -> endswith(name, ".json"), readdir(directory))), +) @testset "$(file)" for file in files groups = JSON.parsefile(joinpath(directory, file)) @testset "$(group["description"])" for group in groups @@ -78,6 +83,14 @@ end schema_dialect, "http://localhost:1234/$name/root", ) + # JSON Schema requires ECMA-262 `pattern` semantics; PCRE differs by default. + _test_compiled_draft( + joinpath(suite, name, "optional"), + retriever, + schema_dialect, + "http://localhost:1234/$name/root"; + files = ["ecmascript-regex.json"], + ) end end end From 62cc9973a568c254526d73a3d74cb2ecd5577be0 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 30 Oct 2026 22:05:46 -0600 Subject: [PATCH 2/3] fix(schema_engine): read pattern escapes as tokens and translate `.` - Read each escape as one token. An escaped `[` or `]` no longer changes the class state, so `^\[\s\]$` and `^[\]\s]$` match as in ECMA-262. - Write `\s` and `\S` as explicit ranges of the ECMA-262 white-space set. The ranges work inside a class, so `[\S]` and `[^\S]` are correct. - Translate an unescaped `.` outside a class to exclude every LineTerminator, `[]` to an empty match, and `[^]` to any code point. --- src/schema_engine/dialects.jl | 172 ++++++++++++++++++++++++--------- test/schema_engine/compiled.jl | 10 ++ 2 files changed, 134 insertions(+), 48 deletions(-) diff --git a/src/schema_engine/dialects.jl b/src/schema_engine/dialects.jl index 736b843..220590a 100644 --- a/src/schema_engine/dialects.jl +++ b/src/schema_engine/dialects.jl @@ -288,17 +288,117 @@ function _ecma_property(property::AbstractString) return String(property) end -# ECMA-262 `\s`: WhiteSpace (tab, vertical tab, form feed, every Zs, U+FEFF) and -# LineTerminator (LF, CR, U+2028, U+2029). PCRE without UCP matches only the ASCII ones. -# Julia compiles with ALT_BSUX, so code points use the JavaScript `\\uhhhh` form. -const ECMA_WHITESPACE = "\\t\\n\\x0B\\f\\r\\p{Zs}\\u2028\\u2029\\uFEFF" +# ECMA-262 `\s`: WhiteSpace (tab, vertical tab, form feed, U+FEFF, and the Unicode 16 +# Zs code points) and LineTerminator (LF, CR, U+2028, U+2029). PCRE without UCP knows +# only the ASCII ones, and cannot remove a set inside a class, so `\s` and `\S` are +# written as explicit ranges that work inside and outside a class. +const ECMA_WHITESPACE_RANGES = ( + 0x09 => 0x0d, + 0x20 => 0x20, + 0xa0 => 0xa0, + 0x1680 => 0x1680, + 0x2000 => 0x200a, + 0x2028 => 0x2029, + 0x202f => 0x202f, + 0x205f => 0x205f, + 0x3000 => 0x3000, + 0xfeff => 0xfeff, +) + +function _ecma_complement(ranges) + output = Pair{UInt32,UInt32}[] + first_free = UInt32(0) + for (low, high) in ranges + low > first_free && push!(output, first_free => UInt32(low - 1)) + first_free = UInt32(high + 1) + end + push!(output, first_free => UInt32(0x10ffff)) + return output +end -function _ecma_whitespace(escaped::Char, in_class::Bool) +# Julia compiles with ALT_BSUX, so BMP code points use the JavaScript `\uhhhh` form. +# PCRE has no escape for an astral code point there, so it is written as a literal. +function _ecma_code_point(code::Integer) + if code <= 0xff + return "\\x" * string(code; base = 16, pad = 2) + elseif code <= 0xffff + return "\\u" * string(code; base = 16, pad = 4) + end + return string(Char(code)) +end + +function _ecma_class_body(ranges) + io = IOBuffer() + for (low, high) in ranges + write(io, _ecma_code_point(low)) + low == high || write(io, '-', _ecma_code_point(high)) + end + return String(take!(io)) +end + +const ECMA_WHITESPACE = _ecma_class_body(ECMA_WHITESPACE_RANGES) +const ECMA_NON_WHITESPACE = _ecma_class_body(_ecma_complement(ECMA_WHITESPACE_RANGES)) +# ECMA-262 `.` matches any code point except a LineTerminator. +const ECMA_DOT = "[^\\n\\r\\u2028\\u2029]" + +function _ecma_write_set(io::IO, body::AbstractString, in_class::Bool) if in_class - # PCRE cannot subtract a set inside a class, so `[\S]` keeps its ASCII meaning. - return escaped == 's' ? ECMA_WHITESPACE : "\\S" + write(io, body) + else + write(io, '[', body, ']') + end + return +end + +function _ecma_property_escape!(io::IO, pattern::AbstractString, i::Int) + property_type = pattern[i] + brace = nextind(pattern, i) + if brace > lastindex(pattern) || pattern[brace] != '{' + write(io, '\\', property_type) + return brace + end + closing = findnext('}', pattern, nextind(pattern, brace)) + if closing === nothing + write(io, '\\', property_type) + return brace + end + property = SubString(pattern, nextind(pattern, brace), prevind(pattern, closing)) + write(io, '\\', property_type, '{', _ecma_property(property), '}') + return nextind(pattern, closing) +end + +# `i` is the index of the escaped character. The pair is consumed here, so an escaped +# `[`, `]`, or `.` never reaches the class tracking in `_ecma_pattern`. +function _ecma_escape!(io::IO, pattern::AbstractString, i::Int, in_class::Bool) + escaped = pattern[i] + if escaped == 's' + _ecma_write_set(io, ECMA_WHITESPACE, in_class) + elseif escaped == 'S' + _ecma_write_set(io, ECMA_NON_WHITESPACE, in_class) + elseif escaped == 'p' || escaped == 'P' + return _ecma_property_escape!(io, pattern, i) + else + write(io, '\\', escaped) end - return escaped == 's' ? "[$ECMA_WHITESPACE]" : "[^$ECMA_WHITESPACE]" + return nextind(pattern, i) +end + +# ECMA-262 reads a `]` right after `[` or `[^` as the end of the class: `[]` matches +# nothing and `[^]` matches any code point. PCRE reads that `]` as a literal. +function _ecma_class_start!(io::IO, pattern::AbstractString, i::Int) + stop = lastindex(pattern) + after = i + empty_class = "(?!)" + if i <= stop && pattern[i] == '^' + after = nextind(pattern, i) + empty_class = "(?s:.)" + end + if after <= stop && pattern[after] == ']' + write(io, empty_class) + return nextind(pattern, after), false + end + write(io, '[') + return i, true end function _ecma_pattern(pattern::AbstractString) @@ -307,45 +407,20 @@ function _ecma_pattern(pattern::AbstractString) i = firstindex(pattern) stop = lastindex(pattern) while i <= stop - if pattern[i] != '\\' - pattern[i] == '[' && (in_class = true) - pattern[i] == ']' && (in_class = false) - write(io, pattern[i]) - i = nextind(pattern, i) - continue - end - slashes = 0 - while i <= stop && pattern[i] == '\\' - slashes += 1 - i = nextind(pattern, i) - end - for _ in 1:slashes - write(io, '\\') + char = pattern[i] + i = nextind(pattern, i) + if char == '\\' && i <= stop + i = _ecma_escape!(io, pattern, i, in_class) + elseif in_class + in_class = char != ']' + write(io, char) + elseif char == '[' + i, in_class = _ecma_class_start!(io, pattern, i) + elseif char == '.' + write(io, ECMA_DOT) + else + write(io, char) end - isodd(slashes) || continue - i <= stop || continue - if pattern[i] in ('s', 'S') - seek(io, position(io) - 1) - truncate(io, position(io)) - write(io, _ecma_whitespace(pattern[i], in_class)) - i = nextind(pattern, i) - continue - end - pattern[i] in ('p', 'P') || continue - property_type = pattern[i] - brace = nextind(pattern, i) - brace <= stop && pattern[brace] == '{' || continue - closing = findnext('}', pattern, nextind(pattern, brace)) - closing === nothing && continue - property = SubString( - pattern, - nextind(pattern, brace), - prevind(pattern, closing), - ) - seek(io, position(io) - 1) - truncate(io, position(io)) - write(io, '\\', property_type, '{', _ecma_property(property), '}') - i = nextind(pattern, closing) end return String(take!(io)) end @@ -355,5 +430,6 @@ end # matches Unicode properties, which needs only UTF mode. const ECMA_COMPILE_OPTIONS = (Base.DEFAULT_COMPILER_OPTS & ~Base.PCRE.UCP) | Base.PCRE.DOLLAR_ENDONLY -_ecma_regex(pattern::AbstractString) = - Regex(_ecma_pattern(pattern), ECMA_COMPILE_OPTIONS, Base.DEFAULT_MATCH_OPTS) +function _ecma_regex(pattern::AbstractString) + return Regex(_ecma_pattern(pattern), ECMA_COMPILE_OPTIONS, Base.DEFAULT_MATCH_OPTS) +end diff --git a/test/schema_engine/compiled.jl b/test/schema_engine/compiled.jl index 5d671b3..861cdb3 100644 --- a/test/schema_engine/compiled.jl +++ b/test/schema_engine/compiled.jl @@ -253,6 +253,16 @@ end @test isvalid(ecma("^\\s\$"), "\u2029") @test !isvalid(ecma("^\\S\$"), "\u2029") @test isvalid(ecma("^[\\s]\$"), "\u3000") + @test !isvalid(ecma("^[\\S]\$"), "\u00a0") + @test isvalid(ecma("^[^\\S]\$"), "\u3000") + @test isvalid(ecma("^\\[\\s\\]\$"), "[ ]") + @test isvalid(ecma("^[\\]\\s]\$"), "]") + @test !isvalid(ecma("^a.b\$"), "a\rb") + @test !isvalid(ecma("^a.b\$"), "a\u2028b") + @test isvalid(ecma("^a.b\$"), "a\U0001f600b") + @test !isvalid(ecma("^a\\.b\$"), "axb") + @test !isvalid(ecma("^a[]\$"), "a") + @test isvalid(ecma("^[^]\$"), "\n") @test isvalid(ecma("^\\p{digit}+\$"), "\u07c0\u0663") @test isvalid(ecma("^\\p{Letter}cole\$"), "école") @test_throws SchemaEngine.CompilationError SchemaEngine.CompiledSchema( From 205d653068e84abd9d253b05b2c018edc38919cd Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Sat, 31 Oct 2026 11:56:08 -0600 Subject: [PATCH 3/3] fix(schema_engine): copy POSIX classes in a pattern class as one unit --- src/schema_engine/dialects.jl | 18 ++++++++++++++++++ test/schema_engine/compiled.jl | 2 ++ 2 files changed, 20 insertions(+) diff --git a/src/schema_engine/dialects.jl b/src/schema_engine/dialects.jl index 220590a..50a8d4e 100644 --- a/src/schema_engine/dialects.jl +++ b/src/schema_engine/dialects.jl @@ -401,6 +401,21 @@ function _ecma_class_start!(io::IO, pattern::AbstractString, i::Int) return i, true end +# PCRE reads `[:name:]` in a class as a POSIX class. Copy it as one unit, so its `]` does +# not end the class tracking. `i` is the index after the `[`. +function _ecma_posix_class!(io::IO, pattern::AbstractString, i::Int) + if i > lastindex(pattern) || pattern[i] != ':' + return i + end + closing = findnext(":]", pattern, nextind(pattern, i)) + bracket = findnext(']', pattern, i) + if closing === nothing || last(closing) != bracket + return i + end + write(io, SubString(pattern, i, bracket)) + return nextind(pattern, bracket) +end + function _ecma_pattern(pattern::AbstractString) io = IOBuffer() in_class = false @@ -414,6 +429,9 @@ function _ecma_pattern(pattern::AbstractString) elseif in_class in_class = char != ']' write(io, char) + if char == '[' + i = _ecma_posix_class!(io, pattern, i) + end elseif char == '[' i, in_class = _ecma_class_start!(io, pattern, i) elseif char == '.' diff --git a/test/schema_engine/compiled.jl b/test/schema_engine/compiled.jl index 861cdb3..f0718a6 100644 --- a/test/schema_engine/compiled.jl +++ b/test/schema_engine/compiled.jl @@ -261,6 +261,8 @@ end @test !isvalid(ecma("^a.b\$"), "a\u2028b") @test isvalid(ecma("^a.b\$"), "a\U0001f600b") @test !isvalid(ecma("^a\\.b\$"), "axb") + @test isvalid(ecma("^[[:alpha:].]\$"), ".") + @test !isvalid(ecma("^[[:alpha:].]\$"), "-") @test !isvalid(ecma("^a[]\$"), "a") @test isvalid(ecma("^[^]\$"), "\n") @test isvalid(ecma("^\\p{digit}+\$"), "\u07c0\u0663")