Rename parser to lexer

Signed-off-by: Marcel Müller <neikos@neikos.email>

benches/asting.rs

@@ -19,7 +19,7 @@ fn asting_benchmark(c: &mut Criterion) {
         parsing.throughput(criterion::Throughput::Bytes(input.len() as u64));
         parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| {
             b.iter(|| {
-                let tokens = nomo::parser::parse(input.clone()).unwrap();
+                let tokens = nomo::lexer::parse(input.clone()).unwrap();
                 let _ast = nomo::ast::parse(tokens.tokens()).unwrap();
             });
         });
@@ -43,7 +43,7 @@ fn asting_nested(c: &mut Criterion) {
         parsing.throughput(criterion::Throughput::Bytes(input.len() as u64));
         parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| {
             b.iter(|| {
-                let tokens = nomo::parser::parse(input.clone()).unwrap();
+                let tokens = nomo::lexer::parse(input.clone()).unwrap();
                 let _ast = nomo::ast::parse(tokens.tokens()).unwrap();
             });
         });

benches/parsing.rs

@@ -18,7 +18,7 @@ fn parsing_benchmark(c: &mut Criterion) {
 
         parsing.throughput(criterion::Throughput::Bytes(input.len() as u64));
         parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| {
-            b.iter(|| nomo::parser::parse(input.clone()).unwrap());
+            b.iter(|| nomo::lexer::parse(input.clone()).unwrap());
         });
     }
 }
@@ -39,7 +39,7 @@ fn parsing_nested(c: &mut Criterion) {
 
         parsing.throughput(criterion::Throughput::Bytes(input.len() as u64));
         parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| {
-            b.iter(|| nomo::parser::parse(input.clone()).unwrap());
+            b.iter(|| nomo::lexer::parse(input.clone()).unwrap());
         });
     }
 }

fuzz/fuzz_targets/fuzz_target_1.rs

@@ -4,7 +4,7 @@ use libfuzzer_sys::Corpus;
 use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: String| -> Corpus {
-    let Ok(parsed) = nomo::parser::parse(data.into()) else {
+    let Ok(parsed) = nomo::lexer::parse(data.into()) else {
         return Corpus::Reject;
     };
 

src/ast/mod.rs

@@ -26,9 +26,9 @@ use winnow::stream::TokenSlice;
 use winnow::token::any;
 
 use crate::SourceSpan;
-use crate::parser::TemplateToken;
+use crate::lexer::TemplateToken;
-use crate::parser::TokenKind;
+use crate::lexer::TokenKind;
-use crate::parser::TokenOperator;
+use crate::lexer::TokenOperator;
 use crate::resume_after_cut;
 use crate::value::NomoValue;
 
@@ -653,8 +653,8 @@ fn parse_literal<'input>(input: &mut Input<'input>) -> Result<TemplateAstExpr<'i
             TokenKind::Literal(literal) => Some(TemplateAstExpr::Literal {
                 source: token.clone(),
                 value: match literal {
-                    crate::parser::TokenLiteral::Bool(bool) => NomoValue::Bool { value: bool },
+                    crate::lexer::TokenLiteral::Bool(bool) => NomoValue::Bool { value: bool },
-                    crate::parser::TokenLiteral::Integer(int) => NomoValue::Integer { value: int },
+                    crate::lexer::TokenLiteral::Integer(int) => NomoValue::Integer { value: int },
                 },
             }),
             _ => None,
@@ -744,7 +744,7 @@ mod tests {
     use crate::ast::parse;
     use crate::ast::parse_block;
     use crate::ast::parse_end;
-    use crate::parser::TokenKind;
+    use crate::lexer::TokenKind;
 
     fn panic_pretty<'a>(
         input: &'_ str,
@@ -762,7 +762,7 @@ mod tests {
     fn check_only_content() {
         let input = "Hello World";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = parse(parsed.tokens()).unwrap();
 
@@ -781,7 +781,7 @@ mod tests {
     fn check_simple_variable_interpolation() {
         let input = "Hello {{= world }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = parse(parsed.tokens()).unwrap();
 
@@ -809,7 +809,7 @@ mod tests {
     fn check_simple_if() {
         let input = "{{ if foo }} Hiii {{ end }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 
@@ -858,7 +858,7 @@ mod tests {
         {{ value }}
         {{ value }}"#;
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = parse(parsed.tokens()).unwrap_err();
 
@@ -876,7 +876,7 @@ mod tests {
         {{= value }}
         "#;
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         insta::assert_debug_snapshot!("simple_if_tokens", parsed);
 
@@ -891,7 +891,7 @@ mod tests {
 
         let input = "{{ foo }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let result = alt((
             parse_end,
@@ -937,7 +937,7 @@ mod tests {
     fn check_empty_if_output() {
         let input = "{{ if foo }}{{ end }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 
@@ -974,7 +974,7 @@ mod tests {
     fn check_if_else() {
         let input = "{{ if foo }} foo {{ else }} bar {{ end }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 
@@ -985,7 +985,7 @@ mod tests {
     fn check_if_else_if() {
         let input = "{{ if foo }} foo {{ else if bar }} bar {{ end }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 
@@ -996,7 +996,7 @@ mod tests {
     fn check_trim_whitespace() {
         let input = "{{ if foo -}} foo {{- else if bar -}} bar {{- end }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 
@@ -1007,7 +1007,7 @@ mod tests {
     fn check_for_loop() {
         let input = "{{ for value in array }} Hi: {{= value }} {{ else }} No Content :C {{ end }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 
@@ -1018,7 +1018,7 @@ mod tests {
     fn check_math_expression() {
         let input = "{{= 5 * 3 + 2 / 3 }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 
@@ -1029,7 +1029,7 @@ mod tests {
     fn check_logical_expression() {
         let input = "{{= true && false || 3 >= 2 && 5 == 2 }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 
@@ -1040,7 +1040,7 @@ mod tests {
     fn check_function_call() {
         let input = "{{= foo(2 * 3, bar(2 + baz)) }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = panic_pretty(input, parse(parsed.tokens()));
 

src/emit/mod.rs

@@ -2,8 +2,8 @@ use std::collections::BTreeMap;
 
 use crate::ast::TemplateAstExpr;
 use crate::input::NomoInput;
-use crate::parser::TemplateToken;
+use crate::lexer::TemplateToken;
-use crate::parser::TokenOperator;
+use crate::lexer::TokenOperator;
 use crate::value::NomoValue;
 
 pub struct EmitMachine {
@@ -171,7 +171,7 @@ fn emit_ast_expr(
             let end_label = machine.reserve_label();
             let mut end_indices = vec![];
 
-            let mut previous_post_whitespace_content: &Option<crate::parser::TemplateToken> = &None;
+            let mut previous_post_whitespace_content: &Option<TemplateToken> = &None;
             let mut previous_jump: Option<LabelSlot> = None;
 
             loop {
@@ -500,7 +500,7 @@ mod tests {
     fn check_simple_variable_interpolation() {
         let input = "Hello {{= world }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = crate::ast::parse(parsed.tokens()).unwrap();
 
@@ -536,7 +536,7 @@ mod tests {
     fn check_if_else_if() {
         let input = "{{ if foo }} foo {{ else if bar }} bar {{ else }} foobar {{ end }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = crate::ast::parse(parsed.tokens()).unwrap();
 
@@ -549,7 +549,7 @@ mod tests {
     fn check_function_call() {
         let input = "{{ if foo(23) }} bar {{ else }} foobar {{ end }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = crate::ast::parse(parsed.tokens()).unwrap();
 

src/eval/mod.rs

@@ -202,12 +202,12 @@ pub fn execute(
                 let right_value = scopes.get(right_slot);
 
                 let result = match op {
-                    crate::parser::TokenOperator::Plus => left_value.try_add(right_value),
+                    crate::lexer::TokenOperator::Plus => left_value.try_add(right_value),
-                    crate::parser::TokenOperator::Minus => left_value.try_sub(right_value),
+                    crate::lexer::TokenOperator::Minus => left_value.try_sub(right_value),
-                    crate::parser::TokenOperator::Times => left_value.try_mul(right_value),
+                    crate::lexer::TokenOperator::Times => left_value.try_mul(right_value),
-                    crate::parser::TokenOperator::Divide => left_value.try_div(right_value),
+                    crate::lexer::TokenOperator::Divide => left_value.try_div(right_value),
-                    crate::parser::TokenOperator::And => left_value.try_and(right_value),
+                    crate::lexer::TokenOperator::And => left_value.try_and(right_value),
-                    crate::parser::TokenOperator::Or => left_value.try_or(right_value),
+                    crate::lexer::TokenOperator::Or => left_value.try_or(right_value),
                     _ => todo!(),
                 };
 
@@ -243,7 +243,7 @@ mod tests {
     fn check_simple_variable_interpolation() {
         let input = "Hello {{= world }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = crate::ast::parse(parsed.tokens()).unwrap();
 
@@ -264,7 +264,7 @@ mod tests {
     fn check_method_call() {
         let input = "Hello {{= foo(world) }}";
 
-        let parsed = crate::parser::parse(input.into()).unwrap();
+        let parsed = crate::lexer::parse(input.into()).unwrap();
 
         let ast = crate::ast::parse(parsed.tokens()).unwrap();
 

src/lexer/mod.rs

@@ -613,7 +613,7 @@ fn ident_terminator<'input>(input: &mut Input<'input>) -> PResult<'input, ()> {
 
 #[cfg(test)]
 mod tests {
-    use crate::parser::parse;
+    use crate::lexer::parse;
 
     #[test]
     fn parse_simple() {

src/lexer/snapshots/nomo__lexer__tests__parse_interpolate_bad-2.snap

@@ -1,5 +1,5 @@
 ---
-source: src/parser/mod.rs
+source: src/lexer/mod.rs
 expression: error.to_report()
 ---
 error: Invalid variable identifier

src/lib.rs

@@ -14,7 +14,7 @@ pub mod emit;
 pub mod eval;
 pub mod functions;
 pub mod input;
-pub mod parser;
+pub mod lexer;
 pub mod value;
 
 #[derive(Debug, Error, Display)]
@@ -22,7 +22,7 @@ pub enum NomoError {
     /// Could not parse the given template
     ParseError {
         #[from]
-        source: parser::ParseFailure,
+        source: lexer::ParseFailure,
     },
     /// Invalid Template
     AstError {
@@ -65,7 +65,7 @@ impl Nomo {
         value: impl Into<NomoInput>,
     ) -> Result<(), NomoError> {
         let source = value.into();
-        let parse = parser::parse(source.clone())?;
+        let parse = lexer::parse(source.clone())?;
         let ast = ast::parse(parse.tokens())?;
 
         let instructions = emit::emit_machine(ast);

tests/checks.rs

@@ -5,7 +5,7 @@ fn check_files() {
     for file in files {
         let input = std::fs::read_to_string(file.unwrap().path()).unwrap();
 
-        let Ok(parsed) = nomo::parser::parse(input.into()) else {
+        let Ok(parsed) = nomo::lexer::parse(input.into()) else {
             continue;
         };
 

tests/file_tests.rs

@@ -39,7 +39,7 @@ fn check_for_input([path]: [&Path; 1]) {
         context.try_insert(k, v).unwrap();
     }
 
-    let parsed = nomo::parser::parse(input.into()).unwrap();
+    let parsed = nomo::lexer::parse(input.into()).unwrap();
 
     let _guard = settings.bind_to_scope();