diff --git a/parser/internal/BUILD b/parser/internal/BUILD index af815588e..c5bb23a4e 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -13,6 +13,7 @@ # limitations under the License. load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") load("//bazel:antlr.bzl", "antlr_cc_library") package(default_visibility = ["//visibility:public"]) @@ -29,3 +30,29 @@ antlr_cc_library( src = "Cel.g4", package = "cel_parser_internal", ) + +cc_library( + name = "lexer", + srcs = ["lexer.cc"], + hdrs = ["lexer.h"], + deps = [ + "//common:source", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_test( + name = "lexer_test", + srcs = ["lexer_test.cc"], + deps = [ + ":lexer", + "//common:source", + "//internal:testing", + ], +) diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc new file mode 100644 index 000000000..db08b644f --- /dev/null +++ b/parser/internal/lexer.cc @@ -0,0 +1,711 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" +#include "absl/container/flat_hash_map.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace cel::parser_internal { + +namespace { + +[[nodiscard]] bool IsIdentTrailing(char32_t c) { + return c <= 0x7f && (absl::ascii_isdigit(static_cast(c)) || + absl::ascii_isalpha(static_cast(c)) || c == '_'); +} + +[[nodiscard]] bool IsPlusOrMinus(char32_t c) { return c == '+' || c == '-'; } + +[[nodiscard]] const absl::flat_hash_map& +Keywords() { + static const absl::NoDestructor< + absl::flat_hash_map> + kKeywords({ + {"false", TokenType::kFalse}, + {"true", TokenType::kTrue}, + {"null", TokenType::kNull}, + {"in", TokenType::kIn}, + {"as", TokenType::kReservedWord}, + {"break", TokenType::kReservedWord}, + {"const", TokenType::kReservedWord}, + {"continue", TokenType::kReservedWord}, + {"else", TokenType::kReservedWord}, + {"for", TokenType::kReservedWord}, + {"function", TokenType::kReservedWord}, + {"if", TokenType::kReservedWord}, + {"import", TokenType::kReservedWord}, + {"let", TokenType::kReservedWord}, + {"loop", TokenType::kReservedWord}, + {"package", TokenType::kReservedWord}, + {"namespace", TokenType::kReservedWord}, + {"return", TokenType::kReservedWord}, + {"var", TokenType::kReservedWord}, + {"void", TokenType::kReservedWord}, + {"while", TokenType::kReservedWord}, + }); + return *kKeywords; +} + +} // namespace + +std::string_view TokenTypeToString(TokenType type) { + switch (type) { + case TokenType::kError: + return "error"; + case TokenType::kEnd: + return "end"; + case TokenType::kWhitespace: + return "whitespace"; + case TokenType::kComment: + return "comment"; + case TokenType::kNull: + return "null"; + case TokenType::kFalse: + return "false"; + case TokenType::kTrue: + return "true"; + case TokenType::kIn: + return "in"; + case TokenType::kReservedWord: + return "reserved_word"; + case TokenType::kInt: + return "int"; + case TokenType::kUint: + return "uint"; + case TokenType::kFloat: + return "float"; + case TokenType::kString: + return "string"; + case TokenType::kBytes: + return "bytes"; + case TokenType::kIdent: + return "ident"; + case TokenType::kLeftBracket: + return "["; + case TokenType::kRightBracket: + return "]"; + case TokenType::kLeftBrace: + return "{"; + case TokenType::kRightBrace: + return "}"; + case TokenType::kLeftParen: + return "("; + case TokenType::kRightParen: + return ")"; + case TokenType::kDot: + return "."; + case TokenType::kComma: + return ","; + case TokenType::kMinus: + return "-"; + case TokenType::kPlus: + return "+"; + case TokenType::kAsterisk: + return "*"; + case TokenType::kSlash: + return "/"; + case TokenType::kPercent: + return "%"; + case TokenType::kQuestion: + return "?"; + case TokenType::kColon: + return ":"; + case TokenType::kExclamation: + return "!"; + case TokenType::kEqual: + return "="; + case TokenType::kEqualEqual: + return "=="; + case TokenType::kExclamationEqual: + return "!="; + case TokenType::kLess: + return "<"; + case TokenType::kLessEqual: + return "<="; + case TokenType::kGreater: + return ">"; + case TokenType::kGreaterEqual: + return ">="; + case TokenType::kLogicalAnd: + return "&&"; + case TokenType::kLogicalOr: + return "||"; + default: + return ""; + } +} + +Token Lexer::Lex() { + int32_t start = GetPosition(); + if (ABSL_PREDICT_FALSE(position_ >= content_.size())) { + at_end_ = true; + done_ = true; + return MakeToken(TokenType::kEnd, start, start); + } + char32_t c = content_.at(position_); + switch (c) { + case '\f': + ABSL_FALLTHROUGH_INTENDED; + case '\v': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\n': + ABSL_FALLTHROUGH_INTENDED; + case ' ': { + ConsumeWhitespace(); + return MakeToken(TokenType::kWhitespace, start, GetPosition()); + } + case '.': { + if (position_ + 1 < content_.size() && + content_.at(position_ + 1) <= 0x7f && + absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { + return ConsumeNumericLiteral(); + } + Advance(1); + return MakeToken(TokenType::kDot, start, GetPosition()); + } + case ',': { + Advance(1); + return MakeToken(TokenType::kComma, start, GetPosition()); + } + case '!': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kExclamationEqual, start, GetPosition()); + } + return MakeToken(TokenType::kExclamation, start, GetPosition()); + } + case '?': { + Advance(1); + return MakeToken(TokenType::kQuestion, start, GetPosition()); + } + case '(': { + Advance(1); + return MakeToken(TokenType::kLeftParen, start, GetPosition()); + } + case ')': { + Advance(1); + return MakeToken(TokenType::kRightParen, start, GetPosition()); + } + case '{': { + Advance(1); + return MakeToken(TokenType::kLeftBrace, start, GetPosition()); + } + case '}': { + Advance(1); + return MakeToken(TokenType::kRightBrace, start, GetPosition()); + } + case '[': { + Advance(1); + return MakeToken(TokenType::kLeftBracket, start, GetPosition()); + } + case ']': { + Advance(1); + return MakeToken(TokenType::kRightBracket, start, GetPosition()); + } + case '=': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kEqualEqual, start, GetPosition()); + } + return MakeToken(TokenType::kEqual, start, GetPosition()); + } + case '<': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kLessEqual, start, GetPosition()); + } + return MakeToken(TokenType::kLess, start, GetPosition()); + } + case '>': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kGreaterEqual, start, GetPosition()); + } + return MakeToken(TokenType::kGreater, start, GetPosition()); + } + case ':': { + Advance(1); + return MakeToken(TokenType::kColon, start, GetPosition()); + } + case '%': { + Advance(1); + return MakeToken(TokenType::kPercent, start, GetPosition()); + } + case '+': { + Advance(1); + return MakeToken(TokenType::kPlus, start, GetPosition()); + } + case '-': { + Advance(1); + return MakeToken(TokenType::kMinus, start, GetPosition()); + } + case '*': { + Advance(1); + return MakeToken(TokenType::kAsterisk, start, GetPosition()); + } + case '/': { + Advance(1); + if (Consume('/')) { + ConsumeLine(); + return MakeToken(TokenType::kComment, start, GetPosition()); + } + return MakeToken(TokenType::kSlash, start, GetPosition()); + } + case '&': { + Advance(1); + if (Consume('&')) { + return MakeToken(TokenType::kLogicalAnd, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '&', expected '&&'"); + } + case '|': { + Advance(1); + if (Consume('|')) { + return MakeToken(TokenType::kLogicalOr, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '|', expected '||'"); + } + case '_': { + return ConsumeIdent(); + } + case '`': { + return ConsumeQuotedIdent(); + } + case '\'': { + return ConsumeStringLiteral(start, '\''); + } + case '"': { + return ConsumeStringLiteral(start, '"'); + } + case 'r': + ABSL_FALLTHROUGH_INTENDED; + case 'R': + ABSL_FALLTHROUGH_INTENDED; + case 'b': + ABSL_FALLTHROUGH_INTENDED; + case 'B': { + if (auto token = ConsumePrefixedStringLiteral(); token.has_value()) { + return *token; + } + break; + } + default: + break; + } + if (c <= 0x7f && absl::ascii_isdigit(static_cast(c))) { + return ConsumeNumericLiteral(); + } + if (c <= 0x7f && absl::ascii_isalpha(static_cast(c))) { + // Root identifiers (the ones starting with a period) are returned as + // a sequence of kDot and kIdent tokens. + return ConsumeIdent(); + } + Advance(1); + return SetError(start, GetPosition(), "unexpected character"); +} + +// Consumes characters up to and including the first occurrence of character `c` +// without interpreting backslashes as escapes. +// Returns true if `c` was found and consumed; false if end of input was +// reached. +bool Lexer::ConsumeUntilAfter(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + for (int32_t pos = position_; pos < content_.size(); ++pos) { + if (content_.at(pos) == c) { + AdvanceProcessingNewLines(pos + 1); + return true; + } + } + AdvanceProcessingNewLines(content_.size()); + return false; +} + +// Consumes characters up to and including the first occurrence of substring `s` +// without interpreting backslashes as escapes (`s` must not contain newlines). +// Returns true if `s` was found and consumed; false if end of input was +// reached. +bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + int32_t pos = position_; + while (pos + static_cast(s.size()) <= content_.size()) { + bool match = true; + for (size_t i = 0; i < s.size(); ++i) { + if (content_.at(pos + static_cast(i)) != s[i]) { + match = false; + break; + } + } + if (match) { + AdvanceProcessingNewLines(pos + static_cast(s.size())); + return true; + } + ++pos; + } + AdvanceProcessingNewLines(content_.size()); + return false; +} + +// Consumes characters up to and including the first occurrence of `c` that is +// not preceded by an odd number of backslash ('\') escape characters. Returns +// true if an unescaped `c` was found and consumed; false if reached EOF. +bool Lexer::ConsumeUntilAfterUnescaped(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + ABSL_DCHECK_NE(c, '\\'); + int32_t pos = position_; + bool escaped = false; + while (pos < content_.size()) { + char32_t cc = content_.at(pos); + if (cc == '\\') { + escaped = !escaped; + } else { + if (cc == c && !escaped) { + AdvanceProcessingNewLines(pos + 1); + return true; + } + escaped = false; + } + ++pos; + } + AdvanceProcessingNewLines(content_.size()); + return false; +} + +// Consumes characters up to and including the first occurrence of substring `s` +// where the first character of `s` is not preceded by an odd number of +// backslashes. Returns true if an unescaped `s` was found and consumed; false +// if reached EOF. +bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + int32_t pos = position_; + bool escaped = false; + while (pos < content_.size()) { + char32_t cc = content_.at(pos); + if (cc == '\\') { + escaped = !escaped; + } else { + if (!escaped && pos + static_cast(s.size()) <= content_.size()) { + bool match = true; + for (size_t j = 0; j < s.size(); ++j) { + if (content_.at(pos + static_cast(j)) != s[j]) { + match = false; + break; + } + } + if (match) { + AdvanceProcessingNewLines(pos + static_cast(s.size())); + return true; + } + } + escaped = false; + } + ++pos; + } + AdvanceProcessingNewLines(content_.size()); + return false; +} + +bool Lexer::MatchString(std::u32string_view s) const { + if (position_ + static_cast(s.size()) > content_.size()) { + return false; + } + for (size_t i = 0; i < s.size(); ++i) { + if (content_.at(position_ + static_cast(i)) != s[i]) { + return false; + } + } + return true; +} + +std::optional Lexer::MatchIf( + absl::FunctionRef predicate) const { + if (position_ < content_.size()) { + char32_t cp = content_.at(position_); + if (predicate(cp)) { + return cp; + } + } + return std::nullopt; +} + +void Lexer::ConsumeLine() { + while (position_ < content_.size()) { + if (content_.at(position_) == '\n') { + Advance(1); + return; + } + Advance(1); + } +} + +void Lexer::ConsumeWhitespace() { + while (position_ < content_.size()) { + char32_t c = content_.at(position_); + switch (c) { + case '\f': + ABSL_FALLTHROUGH_INTENDED; + case '\n': + ABSL_FALLTHROUGH_INTENDED; + case ' ': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\v': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + Advance(1); + break; + default: + return; + } + } +} + +bool Lexer::Consume(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + if (Match(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeIgnoreCase(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + if (MatchIgnoreCase(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + if (MatchString(s)) { + Advance(s.size()); + return true; + } + return false; +} + +std::optional Lexer::ConsumeIf( + absl::FunctionRef predicate) { + std::optional match = MatchIf(predicate); + if (match.has_value()) { + ABSL_DCHECK_NE(*match, '\n'); + Advance(1); + } + return match; +} + +bool Lexer::ConsumeDigits() { + bool advanced = false; + while (position_ < content_.size()) { + char32_t c = content_.at(position_); + if (c > 0x7f || !absl::ascii_isdigit(static_cast(c))) { + break; + } + Advance(1); + advanced = true; + } + return advanced; +} + +bool Lexer::ConsumeHexDigits() { + bool advanced = false; + while (position_ < content_.size()) { + char32_t c = content_.at(position_); + if (c > 0x7f || !absl::ascii_isxdigit(static_cast(c))) { + break; + } + Advance(1); + advanced = true; + } + return advanced; +} + +TokenType Lexer::ConsumeIntegralSuffix() { + if (ConsumeIgnoreCase('u')) { + return TokenType::kUint; + } + return TokenType::kInt; +} + +Token Lexer::ConsumeQuotedIdent() { + int32_t start = GetPosition(); + Advance(1); + if (!ConsumeUntilAfter('`')) { + return SetError(start, GetPosition(), "unterminated quoted identifier"); + } + return MakeToken(TokenType::kIdent, start, GetPosition()); +} + +Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes, + bool is_raw) { + Advance(1); + std::u32string triple_quote(3, quote); + if (ConsumeString(std::u32string_view(triple_quote.data(), 2))) { + if (is_raw ? !ConsumeUntilAfterString(triple_quote) + : !ConsumeUntilAfterUnescapedString(triple_quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, start, + GetPosition()); + } + if (is_raw ? !ConsumeUntilAfter(quote) : !ConsumeUntilAfterUnescaped(quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, start, + GetPosition()); +} + +// Consumes prefixed string and bytes literals. +// Handles the following prefix sequences (case-insensitive for 'r' and 'b'): +// - Raw strings: r"...", r'...', r"""...""", r'''...''' +// - Bytes: b"...", b'...', b"""...""", b'''...''' +// - Raw bytes: br"...", br'...', br"""...""", br'''...''', rb"...", rb'...', +// rb"""...""", rb'''...''' +std::optional Lexer::ConsumePrefixedStringLiteral() { + int32_t start = GetPosition(); + if (position_ >= content_.size()) return std::nullopt; + char32_t c = content_.at(position_); + bool is_bytes = (c == 'b' || c == 'B'); + bool is_raw = (c == 'r' || c == 'R'); + size_t lookahead = 1; + if (position_ + 1 < content_.size()) { + char32_t c2 = content_.at(position_ + 1); + if ((is_bytes && (c2 == 'r' || c2 == 'R')) || + (!is_bytes && (c2 == 'b' || c2 == 'B'))) { + is_bytes = true; + is_raw = true; + lookahead = 2; + } + } + if (position_ + static_cast(lookahead) < content_.size()) { + char32_t quote = content_.at(position_ + static_cast(lookahead)); + if (quote == '"' || quote == '\'') { + Advance(lookahead); + return ConsumeStringLiteral(start, quote, is_bytes, is_raw); + } + } + return std::nullopt; +} + +// Consumes a numeric literal token and returns its TokenType (kInt, kUint, or +// kFloat). Recognizes the following literal formats: +// - Hexadecimal integers (kInt / kUint): 0x1A, 0XFFu, 0x0U +// - Decimal integers (kInt / kUint): 0, 45U, 123456 +// - Floating-point numbers (kFloat): .12345, 1.23, 1e6, 1.5e+10, .5e-3 +Token Lexer::ConsumeNumericLiteral() { + int32_t start = GetPosition(); + char32_t c = content_.at(position_); + bool floating_point = false; + if (c == '.') { + floating_point = true; + Advance(1); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after decimal separator"); + } + } else { + Advance(1); + if (c == '0') { + if (ConsumeIgnoreCase('x')) { + if (!ConsumeHexDigits()) { + return SetError( + start, GetPosition(), + "integral literal missing digits after hexadecimal separator"); + } + auto token_type = ConsumeIntegralSuffix(); + if (ConsumeIf(IsIdentTrailing)) { + return SetError( + start, GetPosition(), + absl::StrCat(TokenTypeToString(token_type), + " literal has unexpected trailing characters")); + } + return MakeToken(token_type, start, GetPosition()); + } + } + static_cast(ConsumeDigits()); + if (position_ < content_.size() && content_.at(position_) == '.' && + position_ + 1 < content_.size() && content_.at(position_ + 1) <= 0x7f && + absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { + floating_point = true; + Advance(1); + static_cast(ConsumeDigits()); + } + } + if (ConsumeIgnoreCase('e')) { + floating_point = true; + static_cast(ConsumeIf(IsPlusOrMinus)); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after exponent separator"); + } + } + auto token_type = + floating_point ? TokenType::kFloat : ConsumeIntegralSuffix(); + if (ConsumeIf(IsIdentTrailing)) { + return SetError( + start, GetPosition(), + absl::StrCat(TokenTypeToString(token_type), + " literal has unexpected trailing characters")); + } + return MakeToken(token_type, start, GetPosition()); +} + +Token Lexer::ConsumeIdent() { + int32_t start = GetPosition(); + while (position_ < content_.size()) { + char32_t c = content_.at(position_); + if (!IsIdentTrailing(c)) { + break; + } + Advance(1); + } + int32_t end = GetPosition(); + std::string word = content_.ToString(start, end); + const auto& keywords = Keywords(); + if (auto it = keywords.find(word); it != keywords.end()) { + return MakeToken(it->second, start, end); + } + return MakeToken(TokenType::kIdent, start, end); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h new file mode 100644 index 000000000..ef166d4bb --- /dev/null +++ b/parser/internal/lexer.h @@ -0,0 +1,286 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/optimization.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" +#include "common/source.h" + +namespace cel::parser_internal { + +enum class TokenType { + kError = 0, + kEnd, + kWhitespace, + kComment, + + // Keywords + kNull, + kFalse, + kTrue, + kIn, + kReservedWord, + + // Literals + kInt, + kUint, + kFloat, + kString, + kBytes, + + // Identifiers (standard bare identifiers and backtick-quoted identifiers). + // Note: The lexer does not validate whether a quoted/escaped identifier is + // source-legal or permitted in its syntactic context. Because + // non-source-legal identifiers are used internally in macros and functions, + // the parser must strictly validate the characters inside quoted identifiers + // and verify that they only appear where permitted (e.g., field selections + // and struct field specifiers). + kIdent, + + // Delimiters + kLeftBracket, // [ + kRightBracket, // ] + kLeftBrace, // { + kRightBrace, // } + kLeftParen, // ( + kRightParen, // ) + + // Operators + kDot, // . + kComma, // , + kMinus, // - + kPlus, // + + kAsterisk, // * + kSlash, // / + kPercent, // % + kQuestion, // ? + kColon, // : + kExclamation, // ! + kEqual, // = + kEqualEqual, // == + kExclamationEqual, // != + kLess, // < + kLessEqual, // <= + kGreater, // > + kGreaterEqual, // >= + kLogicalAnd, // && + kLogicalOr, // || +}; + +ABSL_ATTRIBUTE_PURE_FUNCTION std::string_view TokenTypeToString(TokenType type); + +struct Token final { + TokenType type = TokenType::kError; + int32_t start = 0; + int32_t end = 0; +}; + +struct LexerError final { + int32_t start = 0; + int32_t end = 0; + std::string message; +}; + +// Lexer performs fast tokenization of CEL expression source code. +// +// Responsibilities & Parser Expectations: +// This lexer is designed for speed and does not perform comprehensive semantic +// or syntax validation: +// +// 1. String and Bytes Literal Escape Sequences: +// - For standard single- and double-quoted literals ("..." and '...'), the +// lexer recognizes backslash ('\') only to determine whether the closing +// delimiter ('"' or '\'') is escaped (e.g., \" and \' do not terminate the +// literal, whereas \\" terminates it because the backslash is escaped). +// - For triple-quoted literals ("""...""" and '''...''') and raw literals +// (r"...", r'''...'''), backslashes and escape sequences are not processed +// when locating the closing delimiter. +// - The lexer does NOT validate, decode, or check the syntax of any escape +// sequences (e.g., \n, \r, \t, \xHH, \uHHHH, \U00HHHHHH, \0, or invalid +// escapes like \q). All characters and backslashes within the literal +// boundaries are preserved verbatim in the token's text span. +// - The parser/caller is strictly responsible for validating and unescaping +// all escape sequences and reporting syntax errors for invalid escape +// sequences when converting string and bytes tokens during AST +// construction. +// +// 2. Numeric Literals: +// - Performs only general bounds and format matching for integers and +// floating-point numeric literals. The lexer expects the parser to perform +// final validation and numeric conversion when building the AST. +class Lexer final { + public: + explicit Lexer(const cel::Source& source) + : content_(source.content()), position_(0) { + ABSL_DCHECK_LE(content_.size(), static_cast( + std::numeric_limits::max())); + } + + Lexer(const Lexer&) = delete; + Lexer(Lexer&&) = delete; + Lexer& operator=(const Lexer&) = delete; + Lexer& operator=(Lexer&&) = delete; + + // Scans and returns the next token from the source. + [[nodiscard]] ABSL_ATTRIBUTE_NOINLINE Token Lex(); + + // Inspect the error from the last call to `Lex()` that returned an error + // token. The reference is not guaranteed to be valid after further calls to + // `Lex`. + [[nodiscard]] const LexerError& GetError() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return error_; + } + + [[nodiscard]] int32_t GetPosition() const { return position_; } + + private: + [[nodiscard]] bool Match(char32_t c) const { + return position_ < content_.size() && content_.at(position_) == c; + } + + [[nodiscard]] bool MatchIgnoreCase(char32_t c) const { + if (position_ >= content_.size()) return false; + char32_t cp = content_.at(position_); + return cp <= 0x7f && c <= 0x7f && + absl::ascii_tolower(static_cast(cp)) == + absl::ascii_tolower(static_cast(c)); + } + + void Advance(size_t n) { + ABSL_DCHECK_LE(n, static_cast(content_.size() - position_)); + position_ += static_cast(n); + } + + void AdvanceProcessingNewLines(int32_t end_position) { + ABSL_DCHECK_LE(end_position, content_.size()); + ABSL_DCHECK_GE(end_position, position_); + Advance(static_cast(end_position - position_)); + } + + [[nodiscard]] Token MakeToken(TokenType type, int32_t start, int32_t end) { + if (ABSL_PREDICT_FALSE(at_end_)) { + AtEndTokenCreated(); + } + return Token{.type = type, .start = start, .end = end}; + } + + [[nodiscard]] Token SetError(int32_t start, int32_t end, + std::string message) { + error_ = + LexerError{.start = start, .end = end, .message = std::move(message)}; + return Token{.type = TokenType::kError, .start = start, .end = end}; + } + + void AtEndTokenCreated() { done_ = true; } + + // Consumes characters up to and including the first occurrence of character + // `c` without interpreting backslashes as escapes. Returns true if `c` was + // found and consumed; false if end of input was reached. + [[nodiscard]] bool ConsumeUntilAfter(char32_t c); + + // Consumes characters up to and including the first occurrence of substring + // `s` without interpreting backslashes as escapes (`s` must not contain + // newlines). Returns true if `s` was found and consumed; false if end of + // input was reached. + [[nodiscard]] bool ConsumeUntilAfterString(std::u32string_view s); + + // Consumes characters up to and including the first occurrence of `c` that is + // not preceded by an odd number of backslash ('\') escape characters. Returns + // true if an unescaped `c` was found and consumed; false if reached EOF. + [[nodiscard]] bool ConsumeUntilAfterUnescaped(char32_t c); + + // Consumes characters up to and including the first occurrence of substring + // `s` where the first character of `s` is not preceded by an odd number of + // backslashes. Returns true if an unescaped `s` was found and consumed; false + // if reached EOF. + [[nodiscard]] bool ConsumeUntilAfterUnescapedString(std::u32string_view s); + + [[nodiscard]] bool MatchString(std::u32string_view s) const; + + [[nodiscard]] std::optional MatchIf( + absl::FunctionRef predicate) const; + + void ConsumeLine(); + + void ConsumeWhitespace(); + + [[nodiscard]] bool Consume(char32_t c); + + [[nodiscard]] bool ConsumeIgnoreCase(char32_t c); + + [[nodiscard]] bool ConsumeString(std::u32string_view s); + + [[nodiscard]] std::optional ConsumeIf( + absl::FunctionRef predicate); + + [[nodiscard]] bool ConsumeDigits(); + + [[nodiscard]] bool ConsumeHexDigits(); + + [[nodiscard]] TokenType ConsumeIntegralSuffix(); + + // Consumes a backtick-quoted identifier (`...`) and returns + // TokenType::kIdent. The token text preserves the surrounding backticks so + // the parser can detect quoted identifiers and enforce restrictions on their + // characters and allowed syntactic locations (such as field selections and + // struct field specifiers). + [[nodiscard]] Token ConsumeQuotedIdent(); + + [[nodiscard]] Token ConsumeStringLiteral(int32_t start, char32_t quote, + bool is_bytes = false, + bool is_raw = false); + + // Consumes prefixed string and bytes literals. + // Handles the following prefix sequences (case-insensitive for 'r' and 'b'): + // - Raw strings: r"...", r'...', r"""...""", r'''...''' + // - Bytes: b"...", b'...', b"""...""", b'''...''' + // - Raw bytes: br"...", br'...', br"""...""", br'''...''', rb"...", rb'...', + // rb"""...""", rb'''...''' + [[nodiscard]] std::optional ConsumePrefixedStringLiteral(); + + // Consumes a numeric literal token and returns its TokenType (kInt, kUint, or + // kFloat). Recognizes the following literal formats: + // - Decimal integers (kInt / kUint): 0, 45U, 123456 + // - Hexadecimal integers (kInt / kUint): 0x1A, 0XFFu, 0x0U + // - Floating-point numbers (kFloat): .12345, 1.23, 1e6, 1.5e+10, .5e-3 + [[nodiscard]] Token ConsumeNumericLiteral(); + + // Consumes an identifier token and checks if it matches any reserved + // keywords. + [[nodiscard]] Token ConsumeIdent(); + + cel::SourceContentView content_; + int32_t position_ = 0; + bool at_end_ = false; + bool done_ = false; + LexerError error_; +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc new file mode 100644 index 000000000..ca311baf8 --- /dev/null +++ b/parser/internal/lexer_test.cc @@ -0,0 +1,499 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include + +#include "common/source.h" +#include "internal/testing.h" + +namespace cel::parser_internal { +namespace { + +MATCHER_P3(IsToken, source, expected_type, expected_text, "") { + if (arg.type != expected_type) { + *result_listener << "type is " << TokenTypeToString(arg.type) + << " (expected " << TokenTypeToString(expected_type) + << ")"; + return false; + } + std::string actual_text = source->content().ToString(arg.start, arg.end); + if (actual_text != expected_text) { + *result_listener << "text is '" << actual_text << "' (expected '" + << expected_text << "')"; + return false; + } + return true; +} + +struct LexerTestCase { + std::string_view name; + std::string_view source; + std::vector> expected_tokens; +}; + +using LexerTest = testing::TestWithParam; + +TEST_P(LexerTest, LexesSuccessTokens) { + const LexerTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.source)); + Lexer lexer(*source); + + for (const auto& [type, text] : test_case.expected_tokens) { + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), type, text)); + } + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kEnd, "")); +} + +INSTANTIATE_TEST_SUITE_P( + LexerTest, LexerTest, + testing::ValuesIn({ + {"NullSource", std::string_view(nullptr, 0), {}}, + {"Empty", "", {}}, + {"Whitespace", " \n ", {{TokenType::kWhitespace, " \n "}}}, + {"KeywordsAndIdents", + "null false true in as return foo_bar _foo_bar _ `quoted.ident`", + {{TokenType::kNull, "null"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFalse, "false"}, + {TokenType::kWhitespace, " "}, + {TokenType::kTrue, "true"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIn, "in"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "as"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "return"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`quoted.ident`"}}}, + {"Numbers", + "123 45u 0x1A 3.14 .5 1e6 2.5e-3 45U 0x1Au 0x1AU", + {{TokenType::kInt, "123"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x1A"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "3.14"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, ".5"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "1e6"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "2.5e-3"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45U"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1Au"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1AU"}}}, + {"IntEOF", "123456", {{TokenType::kInt, "123456"}}}, + {"HexIntEOF", "0x1A2B", {{TokenType::kInt, "0x1A2B"}}}, + {"FloatPositiveExponentEOF", "1e+6", {{TokenType::kFloat, "1e+6"}}}, + {"FloatEOF", ".12345", {{TokenType::kFloat, ".12345"}}}, + {"IntDotIdent", + "1.foo", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kIdent, "foo"}}}, + {"IntDotWhitespace", + "1. ", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}}}, + {"IntDotEOF", "1.", {{TokenType::kInt, "1"}, {TokenType::kDot, "."}}}, + {"DotAtEOFBeforeDigit", + std::string_view(".6", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"DotAtEOFBeforeIdent", + std::string_view(".a", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"ZeroNumbers", + "0 0u 0x0", + {{TokenType::kInt, "0"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x0"}}}, + {"StringsAndBytes", + R"("hello" 'world' """ "allowed!" ""also allowed"" \"""also allowed""\" """ r"raw" b"bytes" rb'\x00' '''multi +single''' R"raw_upper" B"bytes_upper" b'''multi +bytes''' br"raw_bytes" `a.b-c/d e` +"\a\b\f\n\r\t\v\"\'\\\?\` \x1A \u00A0 \U0001F600 \012")", + {{TokenType::kString, "\"hello\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'world'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, + "\"\"\" \"allowed!\" \"\"also allowed\"\" \\\"\"\"also " + "allowed\"\"\\\" \"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"raw\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "rb'\\x00'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'''multi\nsingle'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "R\"raw_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "B\"bytes_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b'''multi\nbytes'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "br\"raw_bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`a.b-c/d e`"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kString, + "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\\` \\x1A \\u00A0 \\U0001F600 " + "\\012\""}}}, + {"EmptyStrings", + "\"\" '' \"\"\"\"\"\" '''''' r\"\" r'' r\"\"\"\"\"\" r'''''' b\"\" " + "b'' b\"\"\"\"\"\" b''''''", + {{TokenType::kString, "\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''''''"}}}, + {"OperatorsAndDelimiters", + ". , + - * / % == != < <= > >= && || ! ? : [] { } ( )", + {{TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kPlus, "+"}, + {TokenType::kWhitespace, " "}, + {TokenType::kMinus, "-"}, + {TokenType::kWhitespace, " "}, + {TokenType::kAsterisk, "*"}, + {TokenType::kWhitespace, " "}, + {TokenType::kSlash, "/"}, + {TokenType::kWhitespace, " "}, + {TokenType::kPercent, "%"}, + {TokenType::kWhitespace, " "}, + {TokenType::kEqualEqual, "=="}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamationEqual, "!="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLess, "<"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLessEqual, "<="}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreater, ">"}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreaterEqual, ">="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalAnd, "&&"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalOr, "||"}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamation, "!"}, + {TokenType::kWhitespace, " "}, + {TokenType::kQuestion, "?"}, + {TokenType::kWhitespace, " "}, + {TokenType::kColon, ":"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, + {TokenType::kRightBracket, "]"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightBrace, "}"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftParen, "("}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightParen, ")"}}}, + {"Comments", + "a\n// comment\nb", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kComment, "// comment\n"}, + {TokenType::kIdent, "b"}}}, + {"CommentWithoutTrailingNewlineEOF", + "// comment without trailing newline", + {{TokenType::kComment, "// comment without trailing newline"}}}, + {"CommentAfterTokenWithoutTrailingNewlineEOF", + "a // comment without trailing newline", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, " "}, + {TokenType::kComment, "// comment without trailing newline"}}}, + {"NestedDelimiters", + "(((1 + 2) * [3, {4: 5}])))", + {{TokenType::kLeftParen, "("}, {TokenType::kLeftParen, "("}, + {TokenType::kLeftParen, "("}, {TokenType::kInt, "1"}, + {TokenType::kWhitespace, " "}, {TokenType::kPlus, "+"}, + {TokenType::kWhitespace, " "}, {TokenType::kInt, "2"}, + {TokenType::kRightParen, ")"}, {TokenType::kWhitespace, " "}, + {TokenType::kAsterisk, "*"}, {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, {TokenType::kInt, "3"}, + {TokenType::kComma, ","}, {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, {TokenType::kInt, "4"}, + {TokenType::kColon, ":"}, {TokenType::kWhitespace, " "}, + {TokenType::kInt, "5"}, {TokenType::kRightBrace, "}"}, + {TokenType::kRightBracket, "]"}, {TokenType::kRightParen, ")"}, + {TokenType::kRightParen, ")"}, {TokenType::kRightParen, ")"}}}, + {"NestedCommentsInDelimiters", + "(\n // leading comment\n [\n 1, // inline comment\n {2: 3}\n " + " ]\n)", + {{TokenType::kLeftParen, "("}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kComment, "// leading comment\n"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kInt, "1"}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kComment, "// inline comment\n"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, + {TokenType::kInt, "2"}, + {TokenType::kColon, ":"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "3"}, + {TokenType::kRightBrace, "}"}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kRightBracket, "]"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kRightParen, ")"}}}, + {"ComplexNestedLiteralsAndDelimiters", + "`foo.bar`([\"nested\\\"quote\", r'''raw 'quotes' inside'''], " + "b\"bytes\")", + {{TokenType::kIdent, "`foo.bar`"}, + {TokenType::kLeftParen, "("}, + {TokenType::kLeftBracket, "["}, + {TokenType::kString, "\"nested\\\"quote\""}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r'''raw 'quotes' inside'''"}, + {TokenType::kRightBracket, "]"}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"bytes\""}, + {TokenType::kRightParen, ")"}}}, + }), + [](const testing::TestParamInfo& info) { + return std::string(info.param.name); + }); + +TEST(LexerTest, LineOffsets) { + std::string_view source_text = "a\n// comment\nb"; + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(source_text)); + Lexer lexer(*source); + + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kIdent, "a")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kComment, "// comment\n")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kIdent, "b")); + + auto line_offsets = source->line_offsets(); + ASSERT_GE(line_offsets.size(), 2); + EXPECT_EQ(line_offsets[0], 2); + EXPECT_EQ(line_offsets[1], 13); +} + +TEST(LexerTest, LineOffsetsInStringsAndIdentifiers) { + std::string_view source_text = + "'''multi\nline'''\n\"another\nline\"\n`ident\nhere`"; + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(source_text)); + Lexer lexer(*source); + + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kString, "'''multi\nline'''")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kString, "\"another\nline\"")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kIdent, "`ident\nhere`")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kEnd, "")); + + auto line_offsets = source->line_offsets(); + ASSERT_GE(line_offsets.size(), 5); + EXPECT_EQ(line_offsets[0], 9); + EXPECT_EQ(line_offsets[1], 17); + EXPECT_EQ(line_offsets[2], 26); + EXPECT_EQ(line_offsets[3], 32); + EXPECT_EQ(line_offsets[4], 39); +} + +struct LexerErrorTestCase { + std::string_view source; + std::string_view expected_error_message; + std::string_view expected_error_location; +}; + +using LexerErrorTest = testing::TestWithParam; + +TEST_P(LexerErrorTest, LexesErrorTokenAndStoresError) { + const LexerErrorTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.source)); + Lexer lexer(*source); + Token token = lexer.Lex(); + while (token.type != TokenType::kError && token.type != TokenType::kEnd) { + token = lexer.Lex(); + } + EXPECT_EQ(token.type, TokenType::kError); + EXPECT_EQ(lexer.GetError().message, test_case.expected_error_message); + auto location = source->GetLocation(lexer.GetPosition()); + ASSERT_TRUE(location.has_value()); + EXPECT_EQ(source->DisplayErrorLocation(*location), + test_case.expected_error_location); +} + +INSTANTIATE_TEST_SUITE_P( + ErrorCases, LexerErrorTest, + testing::Values( + LexerErrorTestCase{ + .source = "\"unterminated", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | \"unterminated" + "\n | .............^", + }, + LexerErrorTestCase{ + .source = "0x", + .expected_error_message = + "integral literal missing digits after hexadecimal separator", + .expected_error_location = "\n | 0x" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "@", + .expected_error_message = "unexpected character", + .expected_error_location = "\n | @" + "\n | .^", + }, + LexerErrorTestCase{ + .source = "0x1A_invalid", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_error_location = "\n | 0x1A_invalid" + "\n | .....^", + }, + LexerErrorTestCase{ + .source = "123_invalid", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_error_location = "\n | 123_invalid" + "\n | ....^", + }, + LexerErrorTestCase{ + .source = "1x0", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_error_location = "\n | 1x0" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "2x", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_error_location = "\n | 2x" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "`unterminated quoted", + .expected_error_message = "unterminated quoted identifier", + .expected_error_location = "\n | `unterminated quoted" + "\n | ....................^", + }, + LexerErrorTestCase{ + .source = "'''unterminated multi", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | '''unterminated multi" + "\n | .....................^", + }, + LexerErrorTestCase{ + .source = "r'unterminated raw", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | r'unterminated raw" + "\n | ..................^", + }, + LexerErrorTestCase{ + .source = "b'unterminated bytes", + .expected_error_message = "unterminated bytes literal", + .expected_error_location = "\n | b'unterminated bytes" + "\n | ....................^", + }, + LexerErrorTestCase{ + .source = "1e", + .expected_error_message = + "floating point literal missing digits after exponent " + "separator", + .expected_error_location = "\n | 1e" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "\"😀😀😀😀😀\" ~error", + .expected_error_message = "unexpected character", + .expected_error_location = "\n | \"😀😀😀😀😀\" ~error" + "\n | .．．．．．...^", + })); + +TEST(LexerErrorRecoveryTest, ResumesAfterError) { + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("1e, {2 3}")); + Lexer lexer(*source); + Token token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kError); + EXPECT_EQ(lexer.GetError().message, + "floating point literal missing digits after exponent separator"); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kComma); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kWhitespace); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kLeftBrace); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kInt); + EXPECT_EQ(token.start, 5); + EXPECT_EQ(token.end, 6); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/options.h b/parser/internal/options.h index ec2552204..4066d2619 100644 --- a/parser/internal/options.h +++ b/parser/internal/options.h @@ -15,7 +15,7 @@ #ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ -namespace cel_parser_internal { +namespace cel::parser_internal { inline constexpr int kDefaultErrorRecoveryLimit = 12; inline constexpr int kDefaultMaxRecursionDepth = 32; @@ -23,6 +23,6 @@ inline constexpr int kExpressionSizeCodepointLimit = 100'000; inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = 512; inline constexpr bool kDefaultAddMacroCalls = false; -} // namespace cel_parser_internal +} // namespace cel::parser_internal #endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ diff --git a/parser/options.h b/parser/options.h index 719bed454..22154e1f9 100644 --- a/parser/options.h +++ b/parser/options.h @@ -25,25 +25,25 @@ struct ParserOptions final { // Limit of the number of error recovery attempts made by the ANTLR parser // when processing an input. This limit, when reached, will halt further // parsing of the expression. - int error_recovery_limit = ::cel_parser_internal::kDefaultErrorRecoveryLimit; + int error_recovery_limit = ::cel::parser_internal::kDefaultErrorRecoveryLimit; // Limit on the amount of recursive parse instructions permitted when building // the abstract syntax tree for the expression. This prevents pathological // inputs from causing stack overflows. - int max_recursion_depth = ::cel_parser_internal::kDefaultMaxRecursionDepth; + int max_recursion_depth = ::cel::parser_internal::kDefaultMaxRecursionDepth; // Limit on the number of codepoints in the input string which the parser will // attempt to parse. int expression_size_codepoint_limit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; // Limit on the number of lookahead tokens to consume when attempting to // recover from an error. int error_recovery_token_lookahead_limit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; // Add macro calls to macro_calls list in source_info. - bool add_macro_calls = ::cel_parser_internal::kDefaultAddMacroCalls; + bool add_macro_calls = ::cel::parser_internal::kDefaultAddMacroCalls; // Enable support for optional syntax. bool enable_optional_syntax = false; @@ -76,20 +76,20 @@ using ParserOptions = ::cel::ParserOptions; ABSL_DEPRECATED("Use ParserOptions().error_recovery_limit instead.") inline constexpr int kDefaultErrorRecoveryLimit = - ::cel_parser_internal::kDefaultErrorRecoveryLimit; + ::cel::parser_internal::kDefaultErrorRecoveryLimit; ABSL_DEPRECATED("Use ParserOptions().max_recursion_depth instead.") inline constexpr int kDefaultMaxRecursionDepth = - ::cel_parser_internal::kDefaultMaxRecursionDepth; + ::cel::parser_internal::kDefaultMaxRecursionDepth; ABSL_DEPRECATED("Use ParserOptions().expression_size_codepoint_limit instead.") inline constexpr int kExpressionSizeCodepointLimit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; ABSL_DEPRECATED( "Use ParserOptions().error_recovery_token_lookahead_limit instead.") inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; ABSL_DEPRECATED("Use ParserOptions().add_macro_calls instead.") inline constexpr bool kDefaultAddMacroCalls = - ::cel_parser_internal::kDefaultAddMacroCalls; + ::cel::parser_internal::kDefaultAddMacroCalls; } // namespace google::api::expr::parser