fnmain/jotdown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

lex: eat non special chars separately

Browse source 
let tight loop work as long as there no special characters
This commit is contained in:
Noah Hellman 2023-04-23 12:15:39 +02:00
parent 3701d282ac
commit bdab4f021b
1 changed files with 112 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/lex.rs
View file

@ -163,6 +163,10 @@ impl<'s> Lexer<'s> {
' ' => Nbsp,
_ => Text,
}
} else {
self.eat_while(|c| !is_special(c));
if self.len > 0 {
Text
} else {
match self.eat_char()? {
'\n' => Newline,
@ -217,7 +221,8 @@ impl<'s> Lexer<'s> {
self.eat_char();
Close(BraceHyphen)
} else {
while self.peek_char() == Some('-') && self.peek_char_n(1) != Some('}') {
while self.peek_char() == Some('-') && self.peek_char_n(1) != Some('}')
{
self.eat_char();
}
Seq(Hyphen)
@ -251,6 +256,7 @@ impl<'s> Lexer<'s> {
_ => Text,
}
}
};
Some(Token {
@ -282,6 +288,35 @@ impl<'s> Iterator for Lexer<'s> {
}
}
fn is_special(c: char) -> bool {
matches!(
c,
'\\' | '['
| ']'
| '('
| ')'
| '{'
| '}'
| '*'
| '^'
| '='
| '+'
| '~'
| '_'
| '\''
| '"'
| '-'
| '!'
| '<'
| '|'
| ':'
| '`'
| '.'
| '$'
| '\n'
)
}
#[cfg(test)]
mod test {
use super::Delimiter::*;