Hemera/nomo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename parser to lexer

Browse source 
Signed-off-by: Marcel Müller <neikos@neikos.email>
This commit is contained in:
Marcel Müller 2026-03-15 11:31:40 +01:00
parent 10bcd77040
commit f87f4a0262
11 changed files with 46 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

4
benches/asting.rs
View file

@ -19,7 +19,7 @@ fn asting_benchmark(c: &mut Criterion) {
parsing.throughput(criterion::Throughput::Bytes(input.len() as u64));
parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| {
b.iter(|| {
let tokens = nomo::parser::parse(input.clone()).unwrap();
let tokens = nomo::lexer::parse(input.clone()).unwrap();
let _ast = nomo::ast::parse(tokens.tokens()).unwrap();
});
});
@ -43,7 +43,7 @@ fn asting_nested(c: &mut Criterion) {
parsing.throughput(criterion::Throughput::Bytes(input.len() as u64));
parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| {
b.iter(|| {
let tokens = nomo::parser::parse(input.clone()).unwrap();
let tokens = nomo::lexer::parse(input.clone()).unwrap();
let _ast = nomo::ast::parse(tokens.tokens()).unwrap();
});
});

4
benches/parsing.rs
View file

@ -18,7 +18,7 @@ fn parsing_benchmark(c: &mut Criterion) {
parsing.throughput(criterion::Throughput::Bytes(input.len() as u64));
parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| {
b.iter(|| nomo::parser::parse(input.clone()).unwrap());
b.iter(|| nomo::lexer::parse(input.clone()).unwrap());
});
}
}
@ -39,7 +39,7 @@ fn parsing_nested(c: &mut Criterion) {
parsing.throughput(criterion::Throughput::Bytes(input.len() as u64));
parsing.bench_with_input(BenchmarkId::from_parameter(size), &input, |b, input| {
b.iter(|| nomo::parser::parse(input.clone()).unwrap());
b.iter(|| nomo::lexer::parse(input.clone()).unwrap());
});
}
}

2
fuzz/fuzz_targets/fuzz_target_1.rs
View file

@ -4,7 +4,7 @@ use libfuzzer_sys::Corpus;
use libfuzzer_sys::fuzz_target;
fuzz_target!(|data: String| -> Corpus {
let Ok(parsed) = nomo::parser::parse(data.into()) else {
let Ok(parsed) = nomo::lexer::parse(data.into()) else {
return Corpus::Reject;
};

40
src/ast/mod.rs
View file

@ -26,9 +26,9 @@ use winnow::stream::TokenSlice;
use winnow::token::any;
use crate::SourceSpan;
use crate::parser::TemplateToken;
use crate::parser::TokenKind;
use crate::parser::TokenOperator;
use crate::lexer::TemplateToken;
use crate::lexer::TokenKind;
use crate::lexer::TokenOperator;
use crate::resume_after_cut;
use crate::value::NomoValue;
@ -653,8 +653,8 @@ fn parse_literal<'input>(input: &mut Input<'input>) -> Result<TemplateAstExpr<'i
TokenKind::Literal(literal) => Some(TemplateAstExpr::Literal {
source: token.clone(),
value: match literal {
crate::parser::TokenLiteral::Bool(bool) => NomoValue::Bool { value: bool },
crate::parser::TokenLiteral::Integer(int) => NomoValue::Integer { value: int },
crate::lexer::TokenLiteral::Bool(bool) => NomoValue::Bool { value: bool },
crate::lexer::TokenLiteral::Integer(int) => NomoValue::Integer { value: int },
},
}),
_ => None,
@ -744,7 +744,7 @@ mod tests {
use crate::ast::parse;
use crate::ast::parse_block;
use crate::ast::parse_end;
use crate::parser::TokenKind;
use crate::lexer::TokenKind;
fn panic_pretty<'a>(
input: &'_ str,
@ -762,7 +762,7 @@ mod tests {
fn check_only_content() {
let input = "Hello World";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = parse(parsed.tokens()).unwrap();
@ -781,7 +781,7 @@ mod tests {
fn check_simple_variable_interpolation() {
let input = "Hello {{= world }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = parse(parsed.tokens()).unwrap();
@ -809,7 +809,7 @@ mod tests {
fn check_simple_if() {
let input = "{{ if foo }} Hiii {{ end }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));
@ -858,7 +858,7 @@ mod tests {
{{ value }}
{{ value }}"#;
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = parse(parsed.tokens()).unwrap_err();
@ -876,7 +876,7 @@ mod tests {
{{= value }}
"#;
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
insta::assert_debug_snapshot!("simple_if_tokens", parsed);
@ -891,7 +891,7 @@ mod tests {
let input = "{{ foo }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let result = alt((
parse_end,
@ -937,7 +937,7 @@ mod tests {
fn check_empty_if_output() {
let input = "{{ if foo }}{{ end }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));
@ -974,7 +974,7 @@ mod tests {
fn check_if_else() {
let input = "{{ if foo }} foo {{ else }} bar {{ end }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));
@ -985,7 +985,7 @@ mod tests {
fn check_if_else_if() {
let input = "{{ if foo }} foo {{ else if bar }} bar {{ end }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));
@ -996,7 +996,7 @@ mod tests {
fn check_trim_whitespace() {
let input = "{{ if foo -}} foo {{- else if bar -}} bar {{- end }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));
@ -1007,7 +1007,7 @@ mod tests {
fn check_for_loop() {
let input = "{{ for value in array }} Hi: {{= value }} {{ else }} No Content :C {{ end }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));
@ -1018,7 +1018,7 @@ mod tests {
fn check_math_expression() {
let input = "{{= 5 * 3 + 2 / 3 }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));
@ -1029,7 +1029,7 @@ mod tests {
fn check_logical_expression() {
let input = "{{= true && false || 3 >= 2 && 5 == 2 }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));
@ -1040,7 +1040,7 @@ mod tests {
fn check_function_call() {
let input = "{{= foo(2 * 3, bar(2 + baz)) }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = panic_pretty(input, parse(parsed.tokens()));

12
src/emit/mod.rs
View file

@ -2,8 +2,8 @@ use std::collections::BTreeMap;
use crate::ast::TemplateAstExpr;
use crate::input::NomoInput;
use crate::parser::TemplateToken;
use crate::parser::TokenOperator;
use crate::lexer::TemplateToken;
use crate::lexer::TokenOperator;
use crate::value::NomoValue;
pub struct EmitMachine {
@ -171,7 +171,7 @@ fn emit_ast_expr(
let end_label = machine.reserve_label();
let mut end_indices = vec![];
let mut previous_post_whitespace_content: &Option<crate::parser::TemplateToken> = &None;
let mut previous_post_whitespace_content: &Option<TemplateToken> = &None;
let mut previous_jump: Option<LabelSlot> = None;
loop {
@ -500,7 +500,7 @@ mod tests {
fn check_simple_variable_interpolation() {
let input = "Hello {{= world }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = crate::ast::parse(parsed.tokens()).unwrap();
@ -536,7 +536,7 @@ mod tests {
fn check_if_else_if() {
let input = "{{ if foo }} foo {{ else if bar }} bar {{ else }} foobar {{ end }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = crate::ast::parse(parsed.tokens()).unwrap();
@ -549,7 +549,7 @@ mod tests {
fn check_function_call() {
let input = "{{ if foo(23) }} bar {{ else }} foobar {{ end }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = crate::ast::parse(parsed.tokens()).unwrap();

16
src/eval/mod.rs
View file

@ -202,12 +202,12 @@ pub fn execute(
let right_value = scopes.get(right_slot);
let result = match op {
crate::parser::TokenOperator::Plus => left_value.try_add(right_value),
crate::parser::TokenOperator::Minus => left_value.try_sub(right_value),
crate::parser::TokenOperator::Times => left_value.try_mul(right_value),
crate::parser::TokenOperator::Divide => left_value.try_div(right_value),
crate::parser::TokenOperator::And => left_value.try_and(right_value),
crate::parser::TokenOperator::Or => left_value.try_or(right_value),
crate::lexer::TokenOperator::Plus => left_value.try_add(right_value),
crate::lexer::TokenOperator::Minus => left_value.try_sub(right_value),
crate::lexer::TokenOperator::Times => left_value.try_mul(right_value),
crate::lexer::TokenOperator::Divide => left_value.try_div(right_value),
crate::lexer::TokenOperator::And => left_value.try_and(right_value),
crate::lexer::TokenOperator::Or => left_value.try_or(right_value),
_ => todo!(),
};
@ -243,7 +243,7 @@ mod tests {
fn check_simple_variable_interpolation() {
let input = "Hello {{= world }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = crate::ast::parse(parsed.tokens()).unwrap();
@ -264,7 +264,7 @@ mod tests {
fn check_method_call() {
let input = "Hello {{= foo(world) }}";
let parsed = crate::parser::parse(input.into()).unwrap();
let parsed = crate::lexer::parse(input.into()).unwrap();
let ast = crate::ast::parse(parsed.tokens()).unwrap();

2
src/parser/mod.rs → src/lexer/mod.rs
View file

@ -613,7 +613,7 @@ fn ident_terminator<'input>(input: &mut Input<'input>) -> PResult<'input, ()> {
#[cfg(test)]
mod tests {
use crate::parser::parse;
use crate::lexer::parse;
#[test]
fn parse_simple() {

2
src/parser/snapshots/nomo__parser__tests__parse_interpolate_bad-2.snap → src/lexer/snapshots/nomo__lexer__tests__parse_interpolate_bad-2.snap
View file

@ -1,5 +1,5 @@
---
source: src/parser/mod.rs
source: src/lexer/mod.rs
expression: error.to_report()
---
error: Invalid variable identifier

6
src/lib.rs
View file

@ -14,7 +14,7 @@ pub mod emit;
pub mod eval;
pub mod functions;
pub mod input;
pub mod parser;
pub mod lexer;
pub mod value;
#[derive(Debug, Error, Display)]
@ -22,7 +22,7 @@ pub enum NomoError {
/// Could not parse the given template
ParseError {
#[from]
source: parser::ParseFailure,
source: lexer::ParseFailure,
},
/// Invalid Template
AstError {
@ -65,7 +65,7 @@ impl Nomo {
value: impl Into<NomoInput>,
) -> Result<(), NomoError> {
let source = value.into();
let parse = parser::parse(source.clone())?;
let parse = lexer::parse(source.clone())?;
let ast = ast::parse(parse.tokens())?;
let instructions = emit::emit_machine(ast);

2
tests/checks.rs
View file

@ -5,7 +5,7 @@ fn check_files() {
for file in files {
let input = std::fs::read_to_string(file.unwrap().path()).unwrap();
let Ok(parsed) = nomo::parser::parse(input.into()) else {
let Ok(parsed) = nomo::lexer::parse(input.into()) else {
continue;
};

2
tests/file_tests.rs
View file

@ -39,7 +39,7 @@ fn check_for_input([path]: [&Path; 1]) {
context.try_insert(k, v).unwrap();
}
let parsed = nomo::parser::parse(input.into()).unwrap();
let parsed = nomo::lexer::parse(input.into()).unwrap();
let _guard = settings.bind_to_scope();