Hemera/nomo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add tokenization of function pieces

Browse source 
Signed-off-by: Marcel Müller <neikos@neikos.email>
This commit is contained in:
Marcel Müller 2026-03-13 08:38:58 +01:00
parent b2a97c56db
commit 70f616d60c
1 changed files with 60 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

61
src/parser/mod.rs
View file

@ -210,6 +210,9 @@ pub enum TokenKind {
TrimWhitespace,
WantsOutput,
Ident,
LeftArgList,
RightArgList,
ArgSeperator,
Whitespace,
Invalid,
ConditionalIf,
@ -316,6 +319,9 @@ impl TemplateToken {
trim_whitespace => TokenKind::TrimWhitespace,
wants_output => TokenKind::WantsOutput,
ident => TokenKind::Ident,
left_arg_list => TokenKind::LeftArgList,
right_arg_list => TokenKind::RightArgList,
arg_seperator => TokenKind::ArgSeperator,
whitespace => TokenKind::Whitespace,
invalid => TokenKind::Invalid,
conditional_if => TokenKind::ConditionalIf,
@ -415,7 +421,13 @@ fn parse_interpolate<'input>(input: &mut Input<'input>) -> PResult<'input, Vec<T
fn parse_block_token<'input>(input: &mut Input<'input>) -> PResult<'input, TemplateToken> {
trace(
"parse_block_token",
alt((parse_ident, parse_keyword, parse_whitespace, parse_operator)),
alt((
parse_ident,
parse_function,
parse_keyword,
parse_whitespace,
parse_operator,
)),
)
.parse_next(input)
}
@ -493,6 +505,18 @@ fn parse_whitespace<'input>(input: &mut Input<'input>) -> PResult<'input, Templa
.parse_next(input)
}
fn parse_function<'input>(input: &mut Input<'input>) -> PResult<'input, TemplateToken> {
trace(
"parse_function",
alt((
"(".map(TemplateToken::left_arg_list),
")".map(TemplateToken::right_arg_list),
",".map(TemplateToken::arg_seperator),
)),
)
.parse_next(input)
}
fn parse_ident<'input>(input: &mut Input<'input>) -> PResult<'input, TemplateToken> {
resume_after_cut(
terminated(
@ -581,6 +605,7 @@ fn ident_terminator<'input>(input: &mut Input<'input>) -> PResult<'input, ()> {
alt((
eof.void(),
one_of(('{', '}')).void(),
one_of(('(', ',', ')')).void(),
one_of((' ', '\t', '\r', '\n')).void(),
))
.parse_next(input)
@ -846,4 +871,38 @@ mod tests {
)
"#);
}
#[test]
fn parse_function() {
let input = "{{= foo(23, 4, 3 * 5) }}";
let output = parse(input.into());
insta::assert_debug_snapshot!(output, @r#"
Ok(
ParsedTemplate {
tokens: [
[LeftDelim]"{{" (0..2),
[WantsOutput]"=" (2..3),
[Whitespace]" " (3..4),
[Ident]"foo" (4..7),
[LeftArgList]"(" (7..8),
[Literal(Integer(23))]"23" (8..10),
[ArgSeperator]"," (10..11),
[Whitespace]" " (11..12),
[Literal(Integer(4))]"4" (12..13),
[ArgSeperator]"," (13..14),
[Whitespace]" " (14..15),
[Literal(Integer(3))]"3" (15..16),
[Whitespace]" " (16..17),
[Operator(Times)]"*" (17..18),
[Whitespace]" " (18..19),
[Literal(Integer(5))]"5" (19..20),
[RightArgList]")" (20..21),
[Whitespace]" " (21..22),
[RightDelim]"}}" (22..24),
],
},
)
"#);
}
}