Skip to content

[Pratt Parser] Add a fuzzer comparing ANTLR and Pratt parser outputs and fix uncovered discrepancies #1231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[Pratt Parser] Add a fuzzer comparing ANTLR and Pratt parser outputs and fix uncovered discrepancies #1231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions common/src/main/java/dev/cel/common/internal/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,13 @@ public final class Constants {

public static CelConstant parseInt(String text) throws ParseException {
int base;
if (text.startsWith("-0x")) {
if (text.startsWith("-0x") || text.startsWith("-0X")) {
base = 16;
// Strip off the sign and prefix.
text = text.substring(3);
// Add the sign back.
text = "-" + text;
} else if (text.startsWith("0x")) {
} else if (text.startsWith("0x") || text.startsWith("0X")) {
base = 16;
text = text.substring(2);
if (text.startsWith("-")) {
Expand All @@ -83,7 +83,7 @@ public static CelConstant parseUint(String text) throws ParseException {
throw new ParseException("Unsigned integer literal is missing trailing 'u' suffix", 0);
}
text = text.substring(0, text.length() - 1);
if (text.startsWith("0x")) {
if (text.startsWith("0x") || text.startsWith("0X")) {
base = 16;
text = text.substring(2);
} else {
Expand Down
13 changes: 1 addition & 12 deletions parser/src/main/java/dev/cel/parser/Lexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,6 @@ private void consumeWhitespaceAndComments() {
case '\n':
case ' ':
case '\r':
case 11: // \v
case '\t':
position++;
break;
Expand Down Expand Up @@ -589,18 +588,12 @@ private Token consumeNumericLiteral() {
}
} else {
advance(1);
if (c == '0' && consume('x')) {
if (c == '0' && (consume('x') || consume('X'))) {
if (!consumeHexDigits()) {
return setError(
start, position, "integral literal missing digits after hexadecimal separator");
}
TokenType tokenType = consumeIntegralSuffix();
if (consumeIf(Lexer::isIdentTrailing)) {
return setError(
start,
position,
tokenType.getSymbol() + " literal has unexpected trailing characters");
}
return makeToken(tokenType, start, position);
}
consumeDigits();
Expand All @@ -622,10 +615,6 @@ && isDigit(content.get(position + 1))) {
}
}
TokenType tokenType = floatingPoint ? TokenType.FLOAT : consumeIntegralSuffix();
if (consumeIf(Lexer::isIdentTrailing)) {
return setError(
start, position, tokenType.getSymbol() + " literal has unexpected trailing characters");
}
return makeToken(tokenType, start, position);
}

Expand Down
174 changes: 82 additions & 92 deletions parser/src/main/java/dev/cel/parser/PrattParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import dev.cel.common.CelAbstractSyntaxTree;
import dev.cel.common.CelIssue;
import dev.cel.common.CelOptions;
Expand Down Expand Up @@ -524,21 +523,36 @@ private static CelExpr buildUnaryCall(long id, String function, CelExpr operand)

private CelExpr parseSelectorChain() {
Lexer.TokenType tok = peekToken.type;
CelExpr lhs =
(tok == Lexer.TokenType.EXCLAMATION || tok == Lexer.TokenType.MINUS)
? parseUnaryOps()
: parsePrimary();
if (tok == Lexer.TokenType.EXCLAMATION || tok == Lexer.TokenType.MINUS) {
return parseUnaryOps();
}
return parseMember();
}

private CelExpr parseMember() {
int memberStart = peekToken.start;
boolean startedWithIdentOrDot =
peekToken.type == Lexer.TokenType.DOT
|| peekToken.type == Lexer.TokenType.IDENT
|| peekToken.type == Lexer.TokenType.RESERVED_WORD;
CelExpr lhs = parsePrimary();
boolean canBeStructName =
startedWithIdentOrDot
&& (currentToken.type == Lexer.TokenType.IDENT
|| currentToken.type == Lexer.TokenType.RESERVED_WORD)
&& !isQuotedIdent(currentToken);
currentLhsDepth = 0;
tok = peekToken.type;
Lexer.TokenType tok = peekToken.type;
if (tok == Lexer.TokenType.DOT
|| tok == Lexer.TokenType.LEFT_BRACKET
|| tok == Lexer.TokenType.LEFT_BRACE) {
lhs = parseSelectorChainTail(lhs);
lhs = parseSelectorChainTail(lhs, memberStart, canBeStructName);
}
return lhs;
}

private CelExpr parseSelectorChainTail(CelExpr initialLhs) {
private CelExpr parseSelectorChainTail(
CelExpr initialLhs, int memberStartPosition, boolean canBeStructName) {
CelExpr lhs = initialLhs;
int chainDepth = 0;
while (true) {
Expand All @@ -558,9 +572,7 @@ private CelExpr parseSelectorChainTail(CelExpr initialLhs) {
}
}
Lexer.Token idTok = nextToken();
if (idTok.type != Lexer.TokenType.IDENT
&& idTok.type != Lexer.TokenType.RESERVED_WORD
&& idTok.type != Lexer.TokenType.IN) {
if (idTok.type != Lexer.TokenType.IDENT && idTok.type != Lexer.TokenType.RESERVED_WORD) {
if (idTok.type != Lexer.TokenType.ERROR) {
reportSyntaxError(idTok, "expected identifier after '.'");
}
Expand All @@ -570,11 +582,17 @@ private CelExpr parseSelectorChainTail(CelExpr initialLhs) {
}
boolean isMemberCall = (peekToken.type == Lexer.TokenType.LEFT_PAREN);
String idText = normalizeIdent(idTok, /* allowQuoted= */ !isMemberCall);
if (idText.isEmpty()) {
synchronizeOnDelimiter();
currentLhsDepth = chainDepth;
return ERROR;
}
if (optional) {
long opId = nextId(dotTok);
CelExpr field =
CelExpr.ofConstant(nextId(getLeftmostPosition(lhs)), CelConstant.ofValue(idText));
CelExpr.ofConstant(nextId(memberStartPosition), CelConstant.ofValue(idText));
lhs = buildBinaryCall(opId, Operator.OPTIONAL_SELECT.getFunction(), lhs, field);
canBeStructName = false;
} else if (peekToken.type == Lexer.TokenType.LEFT_PAREN) {
Lexer.Token lparen = nextToken();
long callId = nextId(lparen);
Expand All @@ -584,8 +602,10 @@ private CelExpr parseSelectorChainTail(CelExpr initialLhs) {
expanded.isPresent()
? expanded.get()
: CelExpr.ofCall(callId, Optional.of(lhs), idText, args);
canBeStructName = false;
} else {
lhs = CelExpr.ofSelect(nextId(dotTok), lhs, idText, /* isTestOnly= */ false);
canBeStructName = canBeStructName && !isQuotedIdent(idTok);
}
} else if (tok == Lexer.TokenType.LEFT_BRACKET) {
if (checkRecursion(chainDepth, peekToken)) {
Expand All @@ -607,12 +627,17 @@ private CelExpr parseSelectorChainTail(CelExpr initialLhs) {
String opName =
optional ? Operator.OPTIONAL_INDEX.getFunction() : Operator.INDEX.getFunction();
lhs = buildBinaryCall(opId, opName, lhs, index);
canBeStructName = false;
} else if (tok == Lexer.TokenType.LEFT_BRACE) {
if (!canBeStructName) {
break;
}
String structName = extractStructName(lhs);
if (structName == null) {
break;
}
lhs = parseStruct(nextId(peekToken.start), structName);
canBeStructName = false;
} else {
break;
}
Expand All @@ -622,83 +647,42 @@ private CelExpr parseSelectorChainTail(CelExpr initialLhs) {
}

private CelExpr parseUnaryOps() {
Lexer.Token op = nextToken();
Lexer.TokenType opType = op.type;
if (peekToken.type == Lexer.TokenType.EXCLAMATION || peekToken.type == Lexer.TokenType.MINUS) {
return parseUnaryOpsChain(op);
}

if (opType == Lexer.TokenType.MINUS) {
if (peekToken.type == Lexer.TokenType.INT) {
return parseIntLiteral(nextId(peekToken), /* isNegative= */ true);
}
if (peekToken.type == Lexer.TokenType.FLOAT) {
return parseDoubleLiteral(nextId(peekToken), /* isNegative= */ true);
}
}

if (checkRecursion(0, op)) {
return ERROR;
}

long opId = nextId(op);
recursionDepth++;
CelExpr operand = parseSelectorChain();
recursionDepth--;
if (recursionLimitExceeded) {
return ERROR;
}

String opName =
(opType == Lexer.TokenType.EXCLAMATION)
? Operator.LOGICAL_NOT.getFunction()
: Operator.NEGATE.getFunction();
return buildUnaryCall(opId, opName, operand);
}

private CelExpr parseUnaryOpsChain(Lexer.Token firstOp) {
Lexer.Token firstOp = nextToken();
Lexer.TokenType opType = firstOp.type;
List<UnaryOp> ops = new ArrayList<>();
ops.add(new UnaryOp(firstOp));
while (peekToken.type == Lexer.TokenType.EXCLAMATION
|| peekToken.type == Lexer.TokenType.MINUS) {
while (peekToken.type == opType) {
ops.add(new UnaryOp(nextToken()));
}

boolean hasSolitaryTrailingMinus =
!ops.isEmpty()
&& Iterables.getLast(ops).token.type == Lexer.TokenType.MINUS
&& (ops.size() == 1 || ops.get(ops.size() - 2).token.type != Lexer.TokenType.MINUS);

if (!options.retainRepeatedUnaryOperators()) {
int write = 0;
for (int read = 0; read < ops.size(); ) {
int next = read;
while (next < ops.size() && ops.get(next).token.type == ops.get(read).token.type) {
next++;
}
if ((next - read) % 2 != 0) {
ops.set(write++, ops.get(read));
}
read = next;
if (opType == Lexer.TokenType.MINUS
&& ops.size() == 1
&& (peekToken.type == Lexer.TokenType.INT || peekToken.type == Lexer.TokenType.FLOAT)) {
CelExpr lhs =
(peekToken.type == Lexer.TokenType.INT)
? parseIntLiteral(nextId(peekToken), /* isNegative= */ true)
: parseDoubleLiteral(nextId(peekToken), /* isNegative= */ true);
currentLhsDepth = 0;
Lexer.TokenType tok = peekToken.type;
if (tok == Lexer.TokenType.DOT
|| tok == Lexer.TokenType.LEFT_BRACKET
|| tok == Lexer.TokenType.LEFT_BRACE) {
lhs = parseSelectorChainTail(lhs, firstOp.start, /* canBeStructName= */ false);
}
ops = new ArrayList<>(ops.subList(0, write));
}

for (UnaryOp op : ops) {
op.id = nextId(op.token);
return lhs;
}

boolean isNegativeNumericLiteral =
hasSolitaryTrailingMinus
&& (peekToken.type == Lexer.TokenType.INT || peekToken.type == Lexer.TokenType.FLOAT);
long negativeLiteralOpId = 0;
if (isNegativeNumericLiteral) {
negativeLiteralOpId = Iterables.getLast(ops).id;
ops.remove(ops.size() - 1);
if (!options.retainRepeatedUnaryOperators()) {
if (ops.size() % 2 == 0) {
ops.clear();
} else {
ops = new ArrayList<>(ops.subList(0, 1));
}
}

int chainDepth = 0;
for (UnaryOp op : ops) {
op.id = nextId(op.token);
if (checkRecursion(chainDepth, op.token)) {
return ERROR;
}
Expand All @@ -707,14 +691,20 @@ private CelExpr parseUnaryOpsChain(Lexer.Token firstOp) {

recursionDepth += ops.size();
CelExpr operand;
if (isNegativeNumericLiteral) {
operand =
(peekToken.type == Lexer.TokenType.INT)
? parseIntLiteral(negativeLiteralOpId, /* isNegative= */ true)
: parseDoubleLiteral(negativeLiteralOpId, /* isNegative= */ true);
operand = parseSelectorChainTail(operand);
if (opType == Lexer.TokenType.EXCLAMATION && peekToken.type == Lexer.TokenType.MINUS) {
Lexer.Token minusTok = nextToken();
if (peekToken.type == Lexer.TokenType.INT) {
operand = parseIntLiteral(nextId(peekToken), /* isNegative= */ true);
operand = parseSelectorChainTail(operand, minusTok.start, /* canBeStructName= */ false);
} else if (peekToken.type == Lexer.TokenType.FLOAT) {
operand = parseDoubleLiteral(nextId(peekToken), /* isNegative= */ true);
operand = parseSelectorChainTail(operand, minusTok.start, /* canBeStructName= */ false);
} else {
reportSyntaxError(minusTok, "unexpected '-'");
operand = parseMember();
}
} else {
operand = parseSelectorChain();
operand = parseMember();
}
recursionDepth -= ops.size();

Expand Down Expand Up @@ -1044,6 +1034,13 @@ private String normalizeIdent(Lexer.Token tok, boolean allowQuoted) {
return text;
}

private boolean isQuotedIdent(Lexer.Token tok) {
if (tok.text != null) {
return !tok.text.isEmpty() && tok.text.charAt(0) == '`';
}
return tok.start >= 0 && tok.start < tok.end && content.get(tok.start) == '`';
}

private static boolean isAsciiAlphanumeric(char c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9');
}
Expand All @@ -1066,13 +1063,6 @@ private static boolean isAsciiAlphanumeric(char c) {
return null;
}

private int getLeftmostPosition(CelExpr expr) {
while (expr.exprKind().getKind() == CelExpr.ExprKind.Kind.SELECT) {
expr = expr.select().operand();
}
return getPosition(expr.id());
}

private @Nullable CelMacro lookupMacro(String id, int argCount, boolean receiverStyle) {
if (macros.isEmpty()) {
return null;
Expand Down Expand Up @@ -1188,7 +1178,7 @@ private int countGroupingParentheses() {
int size = content.size();
while (pos < size) {
int c = content.get(pos);
if (c != ' ' && c != '\t' && c != '\n' && c != '\r' && c != '\f' && c != 11) {
if (c != ' ' && c != '\t' && c != '\n' && c != '\r' && c != '\f') {
if (c == '/') {
// A comment might precede another '('.
break;
Expand Down
Loading
Loading