diff --git a/css2/tokenizer.cpp b/css2/tokenizer.cpp index 6999557f..0f03a3e8 100644 --- a/css2/tokenizer.cpp +++ b/css2/tokenizer.cpp @@ -56,6 +56,8 @@ std::string_view to_string(ParseError e) { return "EofInEscapeSequence"; case ParseError::EofInString: return "EofInString"; + case ParseError::InvalidEscapeSequence: + return "InvalidEscapeSequence"; case ParseError::NewlineInString: return "NewlineInString"; } @@ -165,9 +167,27 @@ void Tokenizer::run() { case ';': emit(SemiColonToken{}); continue; + case '<': + if (peek_input(0) == '!' && peek_input(1) == '-' && peek_input(2) == '-') { + emit(CdoToken{}); + pos_ += 3; + continue; + } + + emit(DelimToken{'<'}); + continue; case '[': emit(OpenSquareToken{}); continue; + case '\\': + if (is_valid_escape_sequence('\\', peek_input(0))) { + reconsume_in(State::IdentLike); + continue; + } + + emit(ParseError::InvalidEscapeSequence); + emit(DelimToken{'\\'}); + continue; case ']': emit(CloseSquareToken{}); continue; diff --git a/css2/tokenizer.h b/css2/tokenizer.h index d58b9115..d144a687 100644 --- a/css2/tokenizer.h +++ b/css2/tokenizer.h @@ -35,6 +35,7 @@ enum class ParseError : std::uint8_t { EofInComment, EofInEscapeSequence, EofInString, + InvalidEscapeSequence, NewlineInString, }; diff --git a/css2/tokenizer_test.cpp b/css2/tokenizer_test.cpp index 4b7b9704..c9a39924 100644 --- a/css2/tokenizer_test.cpp +++ b/css2/tokenizer_test.cpp @@ -316,6 +316,7 @@ int main() { s.add_test("at keyword start, but with bad escape", [](etest::IActions &a) { auto output = run_tokenizer(a, "@\\\n"); expect_token(output, DelimToken{'@'}); + expect_error(output, ParseError::InvalidEscapeSequence); expect_token(output, DelimToken{'\\'}); expect_token(output, WhitespaceToken{}); }); @@ -508,6 +509,18 @@ int main() { expect_token(output, IdentToken{"lol"}); }); + s.add_test("<: delim", [](etest::IActions &a) { + auto output = run_tokenizer(a, "