css2: Support tokenizing cdo tokens

robinlinden · Jan 23, 2025 · 95c49f2 · 95c49f2
1 parent b011f65
commit 95c49f2
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 0 deletions.
diff --git a/css2/tokenizer.cpp b/css2/tokenizer.cpp
@@ -165,6 +165,15 @@ void Tokenizer::run() {
                     case ';':
                         emit(SemiColonToken{});
                         continue;
+                    case '<':
+                        if (peek_input(0) == '!' && peek_input(1) == '-' && peek_input(2) == '-') {
+                            emit(CdoToken{});
+                            pos_ += 3;
+                            continue;
+                        }
+
+                        emit(DelimToken{'<'});
+                        continue;
                     case '[':
                         emit(OpenSquareToken{});
                         continue;

diff --git a/css2/tokenizer_test.cpp b/css2/tokenizer_test.cpp
@@ -508,6 +508,18 @@ int main() {
         expect_token(output, IdentToken{"lol"});
     });
 
+    s.add_test("<: delim", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "<hello");
+        expect_token(output, DelimToken{'<'});
+        expect_token(output, IdentToken{"hello"});
+    });
+
+    s.add_test("<: cdo", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "<!--lol");
+        expect_token(output, CdoToken{});
+        expect_token(output, IdentToken{"lol"});
+    });
+
     s.add_test("number: ez", [](etest::IActions &a) {
         auto output = run_tokenizer(a, "0.25");
         expect_token(output, NumberToken{0.25});