From 08b480705b2abb6b03e6d67f6414d83ea564d0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Sun, 8 Mar 2026 20:05:54 +0100 Subject: [PATCH] Also emit TokenKind in debug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marcel Müller --- src/ast/mod.rs | 26 +++--- .../nomo__ast__tests__simple_if_ast.snap | 20 ++--- .../nomo__ast__tests__simple_if_tokens.snap | 74 ++++++++--------- src/parser/mod.rs | 64 +++++++-------- tests/cases/1-parsed@condition.snap | 44 +++++----- tests/cases/1-parsed@identifiers.snap | 82 +++++++++---------- tests/cases/1-parsed@interpolation.snap | 16 ++-- tests/cases/1-parsed@multiple.snap | 30 +++---- tests/cases/1-parsed@simple.snap | 2 +- tests/cases/2-ast@condition.snap | 12 +-- tests/cases/2-ast@identifiers.snap | 22 ++--- tests/cases/2-ast@interpolation.snap | 6 +- tests/cases/2-ast@multiple.snap | 10 +-- tests/cases/2-ast@simple.snap | 2 +- 14 files changed, 205 insertions(+), 205 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 29b2333..6160523 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -456,7 +456,7 @@ mod tests { TemplateAst { root: [ StaticContent( - "Hello World" (0..11), + [Content]"Hello World" (0..11), ), ], } @@ -475,14 +475,14 @@ mod tests { TemplateAst { root: [ StaticContent( - "Hello" (0..5), + [Content]"Hello" (0..5), ), Interpolation { prev_whitespace_content: Some( - " " (5..6), + [Whitespace]" " (5..6), ), expression: VariableAccess( - "world" (10..15), + [Ident]"world" (10..15), ), post_whitespace_content: None, }, @@ -508,20 +508,20 @@ mod tests { if_block: Block { prev_whitespace_content: None, expression: VariableAccess( - "foo" (6..9), + [Ident]"foo" (6..9), ), post_whitespace_content: Some( - " " (12..13), + [Whitespace]" " (12..13), ), }, content: [ StaticContent( - "Hiii" (13..17), + [Content]"Hiii" (13..17), ), ], end_block: Block { prev_whitespace_content: Some( - " " (17..18), + [Whitespace]" " (17..18), ), expression: EndBlock, post_whitespace_content: None, @@ -604,11 +604,11 @@ mod tests { insta::assert_debug_snapshot!(result, @r#" ( [ - "{{" (0..2), - " " (2..3), - "foo" (3..6), - " " (6..7), - "}}" (7..9), + [LeftDelim]"{{" (0..2), + [Whitespace]" " (2..3), + [Ident]"foo" (3..6), + [Whitespace]" " (6..7), + [RightDelim]"}}" (7..9), ], None, [ diff --git a/src/ast/snapshots/nomo__ast__tests__simple_if_ast.snap b/src/ast/snapshots/nomo__ast__tests__simple_if_ast.snap index 0cdc6a6..6f99ca6 100644 --- a/src/ast/snapshots/nomo__ast__tests__simple_if_ast.snap +++ b/src/ast/snapshots/nomo__ast__tests__simple_if_ast.snap @@ -10,10 +10,10 @@ TemplateAst { if_block: Block { prev_whitespace_content: None, expression: VariableAccess( - "foo" (6..9), + [Ident]"foo" (6..9), ), post_whitespace_content: Some( - "\n " (12..25), + [Whitespace]"\n " (12..25), ), }, content: [ @@ -23,24 +23,24 @@ TemplateAst { if_block: Block { prev_whitespace_content: None, expression: VariableAccess( - "bar" (31..34), + [Ident]"bar" (31..34), ), post_whitespace_content: Some( - "\n " (37..54), + [Whitespace]"\n " (37..54), ), }, content: [ StaticContent( - "Hiii" (54..58), + [Content]"Hiii" (54..58), ), ], end_block: Block { prev_whitespace_content: Some( - "\n " (58..71), + [Whitespace]"\n " (58..71), ), expression: EndBlock, post_whitespace_content: Some( - "\n " (80..89), + [Whitespace]"\n " (80..89), ), }, }, @@ -51,7 +51,7 @@ TemplateAst { prev_whitespace_content: None, expression: EndBlock, post_whitespace_content: Some( - "\n\n " (98..108), + [Whitespace]"\n\n " (98..108), ), }, }, @@ -60,10 +60,10 @@ TemplateAst { Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "value" (112..117), + [Ident]"value" (112..117), ), post_whitespace_content: Some( - "\n " (120..129), + [Whitespace]"\n " (120..129), ), }, ], diff --git a/src/ast/snapshots/nomo__ast__tests__simple_if_tokens.snap b/src/ast/snapshots/nomo__ast__tests__simple_if_tokens.snap index db5ba26..ffc7d97 100644 --- a/src/ast/snapshots/nomo__ast__tests__simple_if_tokens.snap +++ b/src/ast/snapshots/nomo__ast__tests__simple_if_tokens.snap @@ -4,42 +4,42 @@ expression: parsed --- ParsedTemplate { tokens: [ - "{{" (0..2), - " " (2..3), - "if" (3..5), - " " (5..6), - "foo" (6..9), - " " (9..10), - "}}" (10..12), - "\n " (12..25), - "{{" (25..27), - " " (27..28), - "if" (28..30), - " " (30..31), - "bar" (31..34), - " " (34..35), - "}}" (35..37), - "\n " (37..54), - "Hiii" (54..58), - "\n " (58..71), - "{{" (71..73), - " " (73..74), - "end" (74..77), - " " (77..78), - "}}" (78..80), - "\n " (80..89), - "{{" (89..91), - " " (91..92), - "end" (92..95), - " " (95..96), - "}}" (96..98), - "\n\n " (98..108), - "{{" (108..110), - "=" (110..111), - " " (111..112), - "value" (112..117), - " " (117..118), - "}}" (118..120), - "\n " (120..129), + [LeftDelim]"{{" (0..2), + [Whitespace]" " (2..3), + [ConditionalIf]"if" (3..5), + [Whitespace]" " (5..6), + [Ident]"foo" (6..9), + [Whitespace]" " (9..10), + [RightDelim]"}}" (10..12), + [Whitespace]"\n " (12..25), + [LeftDelim]"{{" (25..27), + [Whitespace]" " (27..28), + [ConditionalIf]"if" (28..30), + [Whitespace]" " (30..31), + [Ident]"bar" (31..34), + [Whitespace]" " (34..35), + [RightDelim]"}}" (35..37), + [Whitespace]"\n " (37..54), + [Content]"Hiii" (54..58), + [Whitespace]"\n " (58..71), + [LeftDelim]"{{" (71..73), + [Whitespace]" " (73..74), + [End]"end" (74..77), + [Whitespace]" " (77..78), + [RightDelim]"}}" (78..80), + [Whitespace]"\n " (80..89), + [LeftDelim]"{{" (89..91), + [Whitespace]" " (91..92), + [End]"end" (92..95), + [Whitespace]" " (95..96), + [RightDelim]"}}" (96..98), + [Whitespace]"\n\n " (98..108), + [LeftDelim]"{{" (108..110), + [WantsOutput]"=" (110..111), + [Whitespace]" " (111..112), + [Ident]"value" (112..117), + [Whitespace]" " (117..118), + [RightDelim]"}}" (118..120), + [Whitespace]"\n " (120..129), ], } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c80b5ec..6c4f857 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -254,7 +254,7 @@ pub struct TemplateToken { impl std::fmt::Debug for TemplateToken { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.source()) + write!(f, "[{:?}]{:?}", self.kind(), self.source()) } } @@ -495,7 +495,7 @@ mod tests { Ok( ParsedTemplate { tokens: [ - "Hello There" (0..11), + [Content]"Hello There" (0..11), ], }, ) @@ -511,13 +511,13 @@ mod tests { Ok( ParsedTemplate { tokens: [ - "Hello" (0..5), - " " (5..6), - "{{" (6..8), - " " (8..9), - "there" (9..14), - " " (14..15), - "}}" (15..17), + [Content]"Hello" (0..5), + [Whitespace]" " (5..6), + [LeftDelim]"{{" (6..8), + [Whitespace]" " (8..9), + [Ident]"there" (9..14), + [Whitespace]" " (14..15), + [RightDelim]"}}" (15..17), ], }, ) @@ -567,29 +567,29 @@ mod tests { Ok( ParsedTemplate { tokens: [ - "{{" (0..2), - " " (2..3), - "if" (3..5), - " " (5..6), - "true" (6..10), - " " (10..11), - "}}" (11..13), - " " (13..14), - "Hello!" (14..20), - " " (20..21), - "{{" (21..23), - " " (23..24), - "else" (24..28), - " " (28..29), - "}}" (29..31), - " " (31..32), - "Bye" (32..35), - " " (35..36), - "{{" (36..38), - " " (38..39), - "end" (39..42), - " " (42..43), - "}}" (43..45), + [LeftDelim]"{{" (0..2), + [Whitespace]" " (2..3), + [ConditionalIf]"if" (3..5), + [Whitespace]" " (5..6), + [Literal(Bool(true))]"true" (6..10), + [Whitespace]" " (10..11), + [RightDelim]"}}" (11..13), + [Whitespace]" " (13..14), + [Content]"Hello!" (14..20), + [Whitespace]" " (20..21), + [LeftDelim]"{{" (21..23), + [Whitespace]" " (23..24), + [ConditionalElse]"else" (24..28), + [Whitespace]" " (28..29), + [RightDelim]"}}" (29..31), + [Whitespace]" " (31..32), + [Content]"Bye" (32..35), + [Whitespace]" " (35..36), + [LeftDelim]"{{" (36..38), + [Whitespace]" " (38..39), + [End]"end" (39..42), + [Whitespace]" " (42..43), + [RightDelim]"}}" (43..45), ], }, ) diff --git a/tests/cases/1-parsed@condition.snap b/tests/cases/1-parsed@condition.snap index 5d3000b..9a4d638 100644 --- a/tests/cases/1-parsed@condition.snap +++ b/tests/cases/1-parsed@condition.snap @@ -5,27 +5,27 @@ input_file: tests/cases/condition.nomo --- ParsedTemplate { tokens: [ - "{{" (0..2), - " " (2..3), - "if" (3..5), - " " (5..6), - "test" (6..10), - " " (10..11), - "}}" (11..13), - "\n " (13..18), - "Hello World!" (18..30), - "\n" (30..31), - "{{" (31..33), - " " (33..34), - "end" (34..37), - " " (37..38), - "}}" (38..40), - "\n\n" (40..42), - "{{" (42..44), - "=" (44..45), - " " (45..46), - "stuff" (46..51), - " " (51..52), - "}}" (52..54), + [LeftDelim]"{{" (0..2), + [Whitespace]" " (2..3), + [ConditionalIf]"if" (3..5), + [Whitespace]" " (5..6), + [Ident]"test" (6..10), + [Whitespace]" " (10..11), + [RightDelim]"}}" (11..13), + [Whitespace]"\n " (13..18), + [Content]"Hello World!" (18..30), + [Whitespace]"\n" (30..31), + [LeftDelim]"{{" (31..33), + [Whitespace]" " (33..34), + [End]"end" (34..37), + [Whitespace]" " (37..38), + [RightDelim]"}}" (38..40), + [Whitespace]"\n\n" (40..42), + [LeftDelim]"{{" (42..44), + [WantsOutput]"=" (44..45), + [Whitespace]" " (45..46), + [Ident]"stuff" (46..51), + [Whitespace]" " (51..52), + [RightDelim]"}}" (52..54), ], } diff --git a/tests/cases/1-parsed@identifiers.snap b/tests/cases/1-parsed@identifiers.snap index da59d40..66a2da0 100644 --- a/tests/cases/1-parsed@identifiers.snap +++ b/tests/cases/1-parsed@identifiers.snap @@ -5,46 +5,46 @@ input_file: tests/cases/identifiers.nomo --- ParsedTemplate { tokens: [ - "{{" (0..2), - "=" (2..3), - " " (3..4), - "_name" (4..9), - " " (9..10), - "}}" (10..12), - "\n" (12..13), - "{{" (13..15), - "=" (15..16), - " " (16..17), - "a_name" (17..23), - " " (23..24), - "}}" (24..26), - "\n" (26..27), - "{{" (27..29), - "=" (29..30), - " " (30..31), - "1name" (31..36), - " " (36..37), - "}}" (37..39), - "\n" (39..40), - "{{" (40..42), - "=" (42..43), - " " (43..44), - "_name1" (44..50), - " " (50..51), - "}}" (51..53), - "\n" (53..54), - "{{" (54..56), - "=" (56..57), - " " (57..58), - "_namE" (58..63), - " " (63..64), - "}}" (64..66), - "\n" (66..67), - "{{" (67..69), - "=" (69..70), - " " (70..71), - "name1" (71..76), - " " (76..77), - "}}" (77..79), + [LeftDelim]"{{" (0..2), + [WantsOutput]"=" (2..3), + [Whitespace]" " (3..4), + [Ident]"_name" (4..9), + [Whitespace]" " (9..10), + [RightDelim]"}}" (10..12), + [Whitespace]"\n" (12..13), + [LeftDelim]"{{" (13..15), + [WantsOutput]"=" (15..16), + [Whitespace]" " (16..17), + [Ident]"a_name" (17..23), + [Whitespace]" " (23..24), + [RightDelim]"}}" (24..26), + [Whitespace]"\n" (26..27), + [LeftDelim]"{{" (27..29), + [WantsOutput]"=" (29..30), + [Whitespace]" " (30..31), + [Ident]"1name" (31..36), + [Whitespace]" " (36..37), + [RightDelim]"}}" (37..39), + [Whitespace]"\n" (39..40), + [LeftDelim]"{{" (40..42), + [WantsOutput]"=" (42..43), + [Whitespace]" " (43..44), + [Ident]"_name1" (44..50), + [Whitespace]" " (50..51), + [RightDelim]"}}" (51..53), + [Whitespace]"\n" (53..54), + [LeftDelim]"{{" (54..56), + [WantsOutput]"=" (56..57), + [Whitespace]" " (57..58), + [Ident]"_namE" (58..63), + [Whitespace]" " (63..64), + [RightDelim]"}}" (64..66), + [Whitespace]"\n" (66..67), + [LeftDelim]"{{" (67..69), + [WantsOutput]"=" (69..70), + [Whitespace]" " (70..71), + [Ident]"name1" (71..76), + [Whitespace]" " (76..77), + [RightDelim]"}}" (77..79), ], } diff --git a/tests/cases/1-parsed@interpolation.snap b/tests/cases/1-parsed@interpolation.snap index 3b5f8a8..662117b 100644 --- a/tests/cases/1-parsed@interpolation.snap +++ b/tests/cases/1-parsed@interpolation.snap @@ -5,13 +5,13 @@ input_file: tests/cases/interpolation.nomo --- ParsedTemplate { tokens: [ - "Hello! I'm" (0..10), - " " (10..11), - "{{" (11..13), - "=" (13..14), - " " (14..15), - "name" (15..19), - " " (19..20), - "}}" (20..22), + [Content]"Hello! I'm" (0..10), + [Whitespace]" " (10..11), + [LeftDelim]"{{" (11..13), + [WantsOutput]"=" (13..14), + [Whitespace]" " (14..15), + [Ident]"name" (15..19), + [Whitespace]" " (19..20), + [RightDelim]"}}" (20..22), ], } diff --git a/tests/cases/1-parsed@multiple.snap b/tests/cases/1-parsed@multiple.snap index 2bc80df..89c0a2a 100644 --- a/tests/cases/1-parsed@multiple.snap +++ b/tests/cases/1-parsed@multiple.snap @@ -5,20 +5,20 @@ input_file: tests/cases/multiple.nomo --- ParsedTemplate { tokens: [ - "Hi there! My name is" (0..20), - " " (20..21), - "{{" (21..23), - "=" (23..24), - " " (24..25), - "name" (25..29), - " " (29..30), - "}}" (30..32), - " " (32..33), - "{{" (33..35), - "=" (35..36), - " " (36..37), - "lastname" (37..45), - " " (45..46), - "}}" (46..48), + [Content]"Hi there! My name is" (0..20), + [Whitespace]" " (20..21), + [LeftDelim]"{{" (21..23), + [WantsOutput]"=" (23..24), + [Whitespace]" " (24..25), + [Ident]"name" (25..29), + [Whitespace]" " (29..30), + [RightDelim]"}}" (30..32), + [Whitespace]" " (32..33), + [LeftDelim]"{{" (33..35), + [WantsOutput]"=" (35..36), + [Whitespace]" " (36..37), + [Ident]"lastname" (37..45), + [Whitespace]" " (45..46), + [RightDelim]"}}" (46..48), ], } diff --git a/tests/cases/1-parsed@simple.snap b/tests/cases/1-parsed@simple.snap index 6da0af7..1bb543e 100644 --- a/tests/cases/1-parsed@simple.snap +++ b/tests/cases/1-parsed@simple.snap @@ -5,6 +5,6 @@ input_file: tests/cases/simple.nomo --- ParsedTemplate { tokens: [ - "Hello World!" (0..12), + [Content]"Hello World!" (0..12), ], } diff --git a/tests/cases/2-ast@condition.snap b/tests/cases/2-ast@condition.snap index 4a68493..d1a35fe 100644 --- a/tests/cases/2-ast@condition.snap +++ b/tests/cases/2-ast@condition.snap @@ -11,24 +11,24 @@ TemplateAst { if_block: Block { prev_whitespace_content: None, expression: VariableAccess( - "test" (6..10), + [Ident]"test" (6..10), ), post_whitespace_content: Some( - "\n " (13..18), + [Whitespace]"\n " (13..18), ), }, content: [ StaticContent( - "Hello World!" (18..30), + [Content]"Hello World!" (18..30), ), ], end_block: Block { prev_whitespace_content: Some( - "\n" (30..31), + [Whitespace]"\n" (30..31), ), expression: EndBlock, post_whitespace_content: Some( - "\n\n" (40..42), + [Whitespace]"\n\n" (40..42), ), }, }, @@ -37,7 +37,7 @@ TemplateAst { Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "stuff" (46..51), + [Ident]"stuff" (46..51), ), post_whitespace_content: None, }, diff --git a/tests/cases/2-ast@identifiers.snap b/tests/cases/2-ast@identifiers.snap index a383d5c..ddcc251 100644 --- a/tests/cases/2-ast@identifiers.snap +++ b/tests/cases/2-ast@identifiers.snap @@ -8,52 +8,52 @@ TemplateAst { Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "_name" (4..9), + [Ident]"_name" (4..9), ), post_whitespace_content: Some( - "\n" (12..13), + [Whitespace]"\n" (12..13), ), }, Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "a_name" (17..23), + [Ident]"a_name" (17..23), ), post_whitespace_content: Some( - "\n" (26..27), + [Whitespace]"\n" (26..27), ), }, Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "1name" (31..36), + [Ident]"1name" (31..36), ), post_whitespace_content: Some( - "\n" (39..40), + [Whitespace]"\n" (39..40), ), }, Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "_name1" (44..50), + [Ident]"_name1" (44..50), ), post_whitespace_content: Some( - "\n" (53..54), + [Whitespace]"\n" (53..54), ), }, Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "_namE" (58..63), + [Ident]"_namE" (58..63), ), post_whitespace_content: Some( - "\n" (66..67), + [Whitespace]"\n" (66..67), ), }, Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "name1" (71..76), + [Ident]"name1" (71..76), ), post_whitespace_content: None, }, diff --git a/tests/cases/2-ast@interpolation.snap b/tests/cases/2-ast@interpolation.snap index 57a4a5b..272123a 100644 --- a/tests/cases/2-ast@interpolation.snap +++ b/tests/cases/2-ast@interpolation.snap @@ -6,14 +6,14 @@ input_file: tests/cases/interpolation.nomo TemplateAst { root: [ StaticContent( - "Hello! I'm" (0..10), + [Content]"Hello! I'm" (0..10), ), Interpolation { prev_whitespace_content: Some( - " " (10..11), + [Whitespace]" " (10..11), ), expression: VariableAccess( - "name" (15..19), + [Ident]"name" (15..19), ), post_whitespace_content: None, }, diff --git a/tests/cases/2-ast@multiple.snap b/tests/cases/2-ast@multiple.snap index 5fdacd6..3f8b6ea 100644 --- a/tests/cases/2-ast@multiple.snap +++ b/tests/cases/2-ast@multiple.snap @@ -6,23 +6,23 @@ input_file: tests/cases/multiple.nomo TemplateAst { root: [ StaticContent( - "Hi there! My name is" (0..20), + [Content]"Hi there! My name is" (0..20), ), Interpolation { prev_whitespace_content: Some( - " " (20..21), + [Whitespace]" " (20..21), ), expression: VariableAccess( - "name" (25..29), + [Ident]"name" (25..29), ), post_whitespace_content: Some( - " " (32..33), + [Whitespace]" " (32..33), ), }, Interpolation { prev_whitespace_content: None, expression: VariableAccess( - "lastname" (37..45), + [Ident]"lastname" (37..45), ), post_whitespace_content: None, }, diff --git a/tests/cases/2-ast@simple.snap b/tests/cases/2-ast@simple.snap index ff6bdba..48a5a9c 100644 --- a/tests/cases/2-ast@simple.snap +++ b/tests/cases/2-ast@simple.snap @@ -6,7 +6,7 @@ input_file: tests/cases/simple.nomo TemplateAst { root: [ StaticContent( - "Hello World!" (0..12), + [Content]"Hello World!" (0..12), ), ], }