Add ast parsing
Signed-off-by: Marcel Müller <neikos@neikos.email>
This commit is contained in:
parent
c5a2179b9e
commit
f5050e369e
3 changed files with 467 additions and 100 deletions
|
|
@ -1,4 +1,3 @@
|
|||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
|
||||
use annotate_snippets::AnnotationKind;
|
||||
|
|
@ -31,14 +30,12 @@ use winnow::token::rest;
|
|||
use winnow::token::take_until;
|
||||
use winnow::token::take_while;
|
||||
|
||||
use crate::SourceSpan;
|
||||
use crate::resume_after_cut;
|
||||
|
||||
type Input<'input> = Recoverable<LocatingSlice<&'input str>, ParseError>;
|
||||
type PResult<'input, T> = Result<T, ParseError>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SourceSpan {
|
||||
pub range: Range<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ParseFailure {
|
||||
source: Arc<str>,
|
||||
|
|
@ -85,7 +82,7 @@ impl ParseFailure {
|
|||
pub struct ParseError {
|
||||
pub(crate) message: Option<String>,
|
||||
pub(crate) help: Option<String>,
|
||||
pub(crate) span: Option<SourceSpan>,
|
||||
pub(crate) span: Option<crate::SourceSpan>,
|
||||
|
||||
is_fatal: bool,
|
||||
}
|
||||
|
|
@ -190,15 +187,104 @@ impl<'input> ParsedTemplate<'input> {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum TemplateToken<'input> {
|
||||
Content(&'input str),
|
||||
LeftDelim(&'input str),
|
||||
RightDelim(&'input str),
|
||||
WantsOutput(&'input str),
|
||||
Ident(&'input str),
|
||||
Whitespace(&'input str),
|
||||
Invalid(&'input str),
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum TokenKind {
|
||||
Content,
|
||||
LeftDelim,
|
||||
RightDelim,
|
||||
WantsOutput,
|
||||
Ident,
|
||||
Whitespace,
|
||||
Invalid,
|
||||
}
|
||||
|
||||
impl PartialEq<TokenKind> for TemplateToken<'_> {
|
||||
fn eq(&self, other: &TokenKind) -> bool {
|
||||
self.kind == *other
|
||||
}
|
||||
}
|
||||
|
||||
impl winnow::stream::ContainsToken<&'_ TemplateToken<'_>> for TokenKind {
|
||||
fn contains_token(&self, token: &'_ TemplateToken<'_>) -> bool {
|
||||
*self == token.kind
|
||||
}
|
||||
}
|
||||
|
||||
impl winnow::stream::ContainsToken<&'_ TemplateToken<'_>> for &'_ [TokenKind] {
|
||||
fn contains_token(&self, token: &'_ TemplateToken<'_>) -> bool {
|
||||
self.contains(&token.kind)
|
||||
}
|
||||
}
|
||||
|
||||
impl<const LEN: usize> winnow::stream::ContainsToken<&'_ TemplateToken<'_>>
|
||||
for &'_ [TokenKind; LEN]
|
||||
{
|
||||
fn contains_token(&self, token: &'_ TemplateToken<'_>) -> bool {
|
||||
self.contains(&token.kind)
|
||||
}
|
||||
}
|
||||
|
||||
impl<const LEN: usize> winnow::stream::ContainsToken<&'_ TemplateToken<'_>> for [TokenKind; LEN] {
|
||||
fn contains_token(&self, token: &'_ TemplateToken<'_>) -> bool {
|
||||
self.contains(&token.kind)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TemplateToken<'input> {
|
||||
kind: TokenKind,
|
||||
source: &'input str,
|
||||
}
|
||||
|
||||
impl<'input> TemplateToken<'input> {
|
||||
fn content(source: &'input str) -> Self {
|
||||
TemplateToken {
|
||||
kind: TokenKind::Content,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
fn left_delim(source: &'input str) -> Self {
|
||||
TemplateToken {
|
||||
kind: TokenKind::LeftDelim,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
fn right_delim(source: &'input str) -> Self {
|
||||
TemplateToken {
|
||||
kind: TokenKind::RightDelim,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
fn wants_output(source: &'input str) -> Self {
|
||||
TemplateToken {
|
||||
kind: TokenKind::WantsOutput,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
fn ident(source: &'input str) -> Self {
|
||||
TemplateToken {
|
||||
kind: TokenKind::Ident,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
fn whitespace(source: &'input str) -> Self {
|
||||
TemplateToken {
|
||||
kind: TokenKind::Whitespace,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(source: &'input str) -> Self {
|
||||
TemplateToken {
|
||||
kind: TokenKind::Invalid,
|
||||
source,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(input: &str) -> Result<ParsedTemplate<'_>, ParseFailure> {
|
||||
|
|
@ -225,7 +311,7 @@ fn parse_content<'input>(input: &mut Input<'input>) -> PResult<'input, Vec<Templ
|
|||
rest.void(),
|
||||
))
|
||||
.take()
|
||||
.map(TemplateToken::Content)
|
||||
.map(TemplateToken::content)
|
||||
.map(|v| vec![v])
|
||||
.parse_next(input)
|
||||
}
|
||||
|
|
@ -234,7 +320,7 @@ fn parse_interpolate<'input>(
|
|||
input: &mut Input<'input>,
|
||||
) -> PResult<'input, Vec<TemplateToken<'input>>> {
|
||||
let prev_whitespace = opt(parse_whitespace).parse_next(input)?;
|
||||
let left_delim = "{{".map(TemplateToken::LeftDelim).parse_next(input)?;
|
||||
let left_delim = "{{".map(TemplateToken::left_delim).parse_next(input)?;
|
||||
|
||||
let get_tokens = repeat_till(1.., parse_interpolate_token, peek("}}"));
|
||||
let recover = take_until(0.., "}}").void();
|
||||
|
|
@ -242,10 +328,10 @@ fn parse_interpolate<'input>(
|
|||
let (inside_tokens, _): (Vec<_>, _) = get_tokens
|
||||
.resume_after(recover)
|
||||
.with_taken()
|
||||
.map(|(val, taken)| val.unwrap_or_else(|| (vec![TemplateToken::Invalid(taken)], "")))
|
||||
.map(|(val, taken)| val.unwrap_or_else(|| (vec![TemplateToken::invalid(taken)], "")))
|
||||
.parse_next(input)?;
|
||||
|
||||
let right_delim = "}}".map(TemplateToken::RightDelim).parse_next(input)?;
|
||||
let right_delim = "}}".map(TemplateToken::right_delim).parse_next(input)?;
|
||||
let post_whitespace = opt(parse_whitespace).parse_next(input)?;
|
||||
|
||||
let mut tokens = vec![];
|
||||
|
|
@ -271,7 +357,7 @@ fn parse_interpolate_token<'input>(
|
|||
fn parse_whitespace<'input>(input: &mut Input<'input>) -> PResult<'input, TemplateToken<'input>> {
|
||||
trace(
|
||||
"parse_whitespace",
|
||||
multispace1.map(TemplateToken::Whitespace),
|
||||
multispace1.map(TemplateToken::whitespace),
|
||||
)
|
||||
.parse_next(input)
|
||||
}
|
||||
|
|
@ -279,7 +365,7 @@ fn parse_whitespace<'input>(input: &mut Input<'input>) -> PResult<'input, Templa
|
|||
fn parse_ident<'input>(input: &mut Input<'input>) -> PResult<'input, TemplateToken<'input>> {
|
||||
resume_after_cut(
|
||||
terminated(
|
||||
ident.map(TemplateToken::Ident),
|
||||
ident.map(TemplateToken::ident),
|
||||
cut_err(ident_terminator_check),
|
||||
)
|
||||
.context(
|
||||
|
|
@ -290,7 +376,7 @@ fn parse_ident<'input>(input: &mut Input<'input>) -> PResult<'input, TemplateTok
|
|||
bad_ident,
|
||||
)
|
||||
.with_taken()
|
||||
.map(|(val, taken)| val.unwrap_or(TemplateToken::Invalid(taken)))
|
||||
.map(|(val, taken)| val.unwrap_or(TemplateToken::invalid(taken)))
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
|
|
@ -317,58 +403,6 @@ fn ident_terminator<'input>(input: &mut Input<'input>) -> PResult<'input, ()> {
|
|||
.parse_next(input)
|
||||
}
|
||||
|
||||
// This is just like the standard .resume_after(), except we only resume on Cut errors.
|
||||
fn resume_after_cut<Input, Output, Error, ParseNext, ParseRecover>(
|
||||
mut parser: ParseNext,
|
||||
mut recover: ParseRecover,
|
||||
) -> impl Parser<Input, Option<Output>, Error>
|
||||
where
|
||||
Input: Stream + winnow::stream::Recover<Error>,
|
||||
Error: ParserError<Input> + FromRecoverableError<Input, Error>,
|
||||
ParseNext: Parser<Input, Output, Error>,
|
||||
ParseRecover: Parser<Input, (), Error>,
|
||||
{
|
||||
trace("resume_after_cut", move |input: &mut Input| {
|
||||
resume_after_cut_inner(&mut parser, &mut recover, input)
|
||||
})
|
||||
}
|
||||
|
||||
fn resume_after_cut_inner<P, R, I, O, E>(
|
||||
parser: &mut P,
|
||||
recover: &mut R,
|
||||
i: &mut I,
|
||||
) -> winnow::Result<Option<O>, E>
|
||||
where
|
||||
P: Parser<I, O, E>,
|
||||
R: Parser<I, (), E>,
|
||||
I: Stream,
|
||||
I: winnow::stream::Recover<E>,
|
||||
E: ParserError<I> + FromRecoverableError<I, E>,
|
||||
{
|
||||
let token_start = i.checkpoint();
|
||||
let mut err = match parser.parse_next(i) {
|
||||
Ok(o) => {
|
||||
return Ok(Some(o));
|
||||
}
|
||||
Err(e) if e.is_incomplete() || e.is_backtrack() => {
|
||||
return Err(e);
|
||||
}
|
||||
Err(err) => err,
|
||||
};
|
||||
let err_start = i.checkpoint();
|
||||
if recover.parse_next(i).is_ok() {
|
||||
if let Err(err_) = i.record_err(&token_start, &err_start, err) {
|
||||
err = err_;
|
||||
} else {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
i.reset(&err_start);
|
||||
err = E::from_recoverable_error(&token_start, &err_start, i, err);
|
||||
Err(err)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::parser::parse;
|
||||
|
|
@ -382,9 +416,10 @@ mod tests {
|
|||
Ok(
|
||||
ParsedTemplate {
|
||||
tokens: [
|
||||
Content(
|
||||
"Hello There",
|
||||
),
|
||||
TemplateToken {
|
||||
kind: Content,
|
||||
source: "Hello There",
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
|
@ -400,27 +435,34 @@ mod tests {
|
|||
Ok(
|
||||
ParsedTemplate {
|
||||
tokens: [
|
||||
Content(
|
||||
"Hello",
|
||||
),
|
||||
Whitespace(
|
||||
" ",
|
||||
),
|
||||
LeftDelim(
|
||||
"{{",
|
||||
),
|
||||
Whitespace(
|
||||
" ",
|
||||
),
|
||||
Ident(
|
||||
"there",
|
||||
),
|
||||
Whitespace(
|
||||
" ",
|
||||
),
|
||||
RightDelim(
|
||||
"}}",
|
||||
),
|
||||
TemplateToken {
|
||||
kind: Content,
|
||||
source: "Hello",
|
||||
},
|
||||
TemplateToken {
|
||||
kind: Whitespace,
|
||||
source: " ",
|
||||
},
|
||||
TemplateToken {
|
||||
kind: LeftDelim,
|
||||
source: "{{",
|
||||
},
|
||||
TemplateToken {
|
||||
kind: Whitespace,
|
||||
source: " ",
|
||||
},
|
||||
TemplateToken {
|
||||
kind: Ident,
|
||||
source: "there",
|
||||
},
|
||||
TemplateToken {
|
||||
kind: Whitespace,
|
||||
source: " ",
|
||||
},
|
||||
TemplateToken {
|
||||
kind: RightDelim,
|
||||
source: "}}",
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue