diff --git a/benches/asting.rs b/benches/asting.rs index 1528069..800942b 100644 --- a/benches/asting.rs +++ b/benches/asting.rs @@ -19,7 +19,7 @@ fn asting_benchmark(c: &mut Criterion) { parsing.throughput(criterion::Throughput::Bytes(input.len() as u64)); parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| { b.iter(|| { - let tokens = nomo::parser::parse(input.clone()).unwrap(); + let tokens = nomo::lexer::parse(input.clone()).unwrap(); let _ast = nomo::ast::parse(tokens.tokens()).unwrap(); }); }); @@ -43,7 +43,7 @@ fn asting_nested(c: &mut Criterion) { parsing.throughput(criterion::Throughput::Bytes(input.len() as u64)); parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| { b.iter(|| { - let tokens = nomo::parser::parse(input.clone()).unwrap(); + let tokens = nomo::lexer::parse(input.clone()).unwrap(); let _ast = nomo::ast::parse(tokens.tokens()).unwrap(); }); }); diff --git a/benches/parsing.rs b/benches/parsing.rs index 35de7ae..20548e6 100644 --- a/benches/parsing.rs +++ b/benches/parsing.rs @@ -18,7 +18,7 @@ fn parsing_benchmark(c: &mut Criterion) { parsing.throughput(criterion::Throughput::Bytes(input.len() as u64)); parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| { - b.iter(|| nomo::parser::parse(input.clone()).unwrap()); + b.iter(|| nomo::lexer::parse(input.clone()).unwrap()); }); } } @@ -39,7 +39,7 @@ fn parsing_nested(c: &mut Criterion) { parsing.throughput(criterion::Throughput::Bytes(input.len() as u64)); parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| { - b.iter(|| nomo::parser::parse(input.clone()).unwrap()); + b.iter(|| nomo::lexer::parse(input.clone()).unwrap()); }); } } diff --git a/fuzz/fuzz_targets/fuzz_target_1.rs b/fuzz/fuzz_targets/fuzz_target_1.rs index 56024b8..ebef862 100644 --- a/fuzz/fuzz_targets/fuzz_target_1.rs +++ b/fuzz/fuzz_targets/fuzz_target_1.rs @@ -4,7 +4,7 @@ use libfuzzer_sys::Corpus; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: String| -> Corpus { - let Ok(parsed) = nomo::parser::parse(data.into()) else { + let Ok(parsed) = nomo::lexer::parse(data.into()) else { return Corpus::Reject; }; diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 3a94816..bdf3044 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -26,9 +26,9 @@ use winnow::stream::TokenSlice; use winnow::token::any; use crate::SourceSpan; -use crate::parser::TemplateToken; -use crate::parser::TokenKind; -use crate::parser::TokenOperator; +use crate::lexer::TemplateToken; +use crate::lexer::TokenKind; +use crate::lexer::TokenOperator; use crate::resume_after_cut; use crate::value::NomoValue; @@ -653,8 +653,8 @@ fn parse_literal<'input>(input: &mut Input<'input>) -> Result Some(TemplateAstExpr::Literal { source: token.clone(), value: match literal { - crate::parser::TokenLiteral::Bool(bool) => NomoValue::Bool { value: bool }, - crate::parser::TokenLiteral::Integer(int) => NomoValue::Integer { value: int }, + crate::lexer::TokenLiteral::Bool(bool) => NomoValue::Bool { value: bool }, + crate::lexer::TokenLiteral::Integer(int) => NomoValue::Integer { value: int }, }, }), _ => None, @@ -744,7 +744,7 @@ mod tests { use crate::ast::parse; use crate::ast::parse_block; use crate::ast::parse_end; - use crate::parser::TokenKind; + use crate::lexer::TokenKind; fn panic_pretty<'a>( input: &'_ str, @@ -762,7 +762,7 @@ mod tests { fn check_only_content() { let input = "Hello World"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = parse(parsed.tokens()).unwrap(); @@ -781,7 +781,7 @@ mod tests { fn check_simple_variable_interpolation() { let input = "Hello {{= world }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = parse(parsed.tokens()).unwrap(); @@ -809,7 +809,7 @@ mod tests { fn check_simple_if() { let input = "{{ if foo }} Hiii {{ end }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); @@ -858,7 +858,7 @@ mod tests { {{ value }} {{ value }}"#; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = parse(parsed.tokens()).unwrap_err(); @@ -876,7 +876,7 @@ mod tests { {{= value }} "#; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); insta::assert_debug_snapshot!("simple_if_tokens", parsed); @@ -891,7 +891,7 @@ mod tests { let input = "{{ foo }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let result = alt(( parse_end, @@ -937,7 +937,7 @@ mod tests { fn check_empty_if_output() { let input = "{{ if foo }}{{ end }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); @@ -974,7 +974,7 @@ mod tests { fn check_if_else() { let input = "{{ if foo }} foo {{ else }} bar {{ end }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); @@ -985,7 +985,7 @@ mod tests { fn check_if_else_if() { let input = "{{ if foo }} foo {{ else if bar }} bar {{ end }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); @@ -996,7 +996,7 @@ mod tests { fn check_trim_whitespace() { let input = "{{ if foo -}} foo {{- else if bar -}} bar {{- end }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); @@ -1007,7 +1007,7 @@ mod tests { fn check_for_loop() { let input = "{{ for value in array }} Hi: {{= value }} {{ else }} No Content :C {{ end }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); @@ -1018,7 +1018,7 @@ mod tests { fn check_math_expression() { let input = "{{= 5 * 3 + 2 / 3 }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); @@ -1029,7 +1029,7 @@ mod tests { fn check_logical_expression() { let input = "{{= true && false || 3 >= 2 && 5 == 2 }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); @@ -1040,7 +1040,7 @@ mod tests { fn check_function_call() { let input = "{{= foo(2 * 3, bar(2 + baz)) }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = panic_pretty(input, parse(parsed.tokens())); diff --git a/src/emit/mod.rs b/src/emit/mod.rs index 0e8b7af..9d1fdd9 100644 --- a/src/emit/mod.rs +++ b/src/emit/mod.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; use crate::ast::TemplateAstExpr; use crate::input::NomoInput; -use crate::parser::TemplateToken; -use crate::parser::TokenOperator; +use crate::lexer::TemplateToken; +use crate::lexer::TokenOperator; use crate::value::NomoValue; pub struct EmitMachine { @@ -171,7 +171,7 @@ fn emit_ast_expr( let end_label = machine.reserve_label(); let mut end_indices = vec![]; - let mut previous_post_whitespace_content: &Option = &None; + let mut previous_post_whitespace_content: &Option = &None; let mut previous_jump: Option = None; loop { @@ -500,7 +500,7 @@ mod tests { fn check_simple_variable_interpolation() { let input = "Hello {{= world }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = crate::ast::parse(parsed.tokens()).unwrap(); @@ -536,7 +536,7 @@ mod tests { fn check_if_else_if() { let input = "{{ if foo }} foo {{ else if bar }} bar {{ else }} foobar {{ end }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = crate::ast::parse(parsed.tokens()).unwrap(); @@ -549,7 +549,7 @@ mod tests { fn check_function_call() { let input = "{{ if foo(23) }} bar {{ else }} foobar {{ end }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = crate::ast::parse(parsed.tokens()).unwrap(); diff --git a/src/eval/mod.rs b/src/eval/mod.rs index 1bb7425..ee8bfc7 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -202,12 +202,12 @@ pub fn execute( let right_value = scopes.get(right_slot); let result = match op { - crate::parser::TokenOperator::Plus => left_value.try_add(right_value), - crate::parser::TokenOperator::Minus => left_value.try_sub(right_value), - crate::parser::TokenOperator::Times => left_value.try_mul(right_value), - crate::parser::TokenOperator::Divide => left_value.try_div(right_value), - crate::parser::TokenOperator::And => left_value.try_and(right_value), - crate::parser::TokenOperator::Or => left_value.try_or(right_value), + crate::lexer::TokenOperator::Plus => left_value.try_add(right_value), + crate::lexer::TokenOperator::Minus => left_value.try_sub(right_value), + crate::lexer::TokenOperator::Times => left_value.try_mul(right_value), + crate::lexer::TokenOperator::Divide => left_value.try_div(right_value), + crate::lexer::TokenOperator::And => left_value.try_and(right_value), + crate::lexer::TokenOperator::Or => left_value.try_or(right_value), _ => todo!(), }; @@ -243,7 +243,7 @@ mod tests { fn check_simple_variable_interpolation() { let input = "Hello {{= world }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = crate::ast::parse(parsed.tokens()).unwrap(); @@ -264,7 +264,7 @@ mod tests { fn check_method_call() { let input = "Hello {{= foo(world) }}"; - let parsed = crate::parser::parse(input.into()).unwrap(); + let parsed = crate::lexer::parse(input.into()).unwrap(); let ast = crate::ast::parse(parsed.tokens()).unwrap(); diff --git a/src/parser/mod.rs b/src/lexer/mod.rs similarity index 99% rename from src/parser/mod.rs rename to src/lexer/mod.rs index 3e0d80c..a35e096 100644 --- a/src/parser/mod.rs +++ b/src/lexer/mod.rs @@ -613,7 +613,7 @@ fn ident_terminator<'input>(input: &mut Input<'input>) -> PResult<'input, ()> { #[cfg(test)] mod tests { - use crate::parser::parse; + use crate::lexer::parse; #[test] fn parse_simple() { diff --git a/src/parser/snapshots/nomo__parser__tests__parse_interpolate_bad-2.snap b/src/lexer/snapshots/nomo__lexer__tests__parse_interpolate_bad-2.snap similarity index 93% rename from src/parser/snapshots/nomo__parser__tests__parse_interpolate_bad-2.snap rename to src/lexer/snapshots/nomo__lexer__tests__parse_interpolate_bad-2.snap index 9dd6238..9b30aec 100644 --- a/src/parser/snapshots/nomo__parser__tests__parse_interpolate_bad-2.snap +++ b/src/lexer/snapshots/nomo__lexer__tests__parse_interpolate_bad-2.snap @@ -1,5 +1,5 @@ --- -source: src/parser/mod.rs +source: src/lexer/mod.rs expression: error.to_report() --- error: Invalid variable identifier diff --git a/src/lib.rs b/src/lib.rs index c9b8222..7c7ad86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,7 @@ pub mod emit; pub mod eval; pub mod functions; pub mod input; -pub mod parser; +pub mod lexer; pub mod value; #[derive(Debug, Error, Display)] @@ -22,7 +22,7 @@ pub enum NomoError { /// Could not parse the given template ParseError { #[from] - source: parser::ParseFailure, + source: lexer::ParseFailure, }, /// Invalid Template AstError { @@ -65,7 +65,7 @@ impl Nomo { value: impl Into, ) -> Result<(), NomoError> { let source = value.into(); - let parse = parser::parse(source.clone())?; + let parse = lexer::parse(source.clone())?; let ast = ast::parse(parse.tokens())?; let instructions = emit::emit_machine(ast); diff --git a/tests/checks.rs b/tests/checks.rs index 56c736a..893b022 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -5,7 +5,7 @@ fn check_files() { for file in files { let input = std::fs::read_to_string(file.unwrap().path()).unwrap(); - let Ok(parsed) = nomo::parser::parse(input.into()) else { + let Ok(parsed) = nomo::lexer::parse(input.into()) else { continue; }; diff --git a/tests/file_tests.rs b/tests/file_tests.rs index 28e88e9..e01a8e8 100644 --- a/tests/file_tests.rs +++ b/tests/file_tests.rs @@ -39,7 +39,7 @@ fn check_for_input([path]: [&Path; 1]) { context.try_insert(k, v).unwrap(); } - let parsed = nomo::parser::parse(input.into()).unwrap(); + let parsed = nomo::lexer::parse(input.into()).unwrap(); let _guard = settings.bind_to_scope();