fnmain/jotdown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

lex: separate escaped/non-escaped

Browse source
This commit is contained in:
Noah Hellman 2023-04-22 19:01:26 +02:00
parent e877fdbde8
commit 3701d282ac
1 changed files with 102 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/lex.rs
View file

@ -93,7 +93,7 @@ impl<'s> Lexer<'s> {
}
}
/// NOTE: Peeked [`Kind::Text`] tokens are only one char long, they may be longer when
/// NOTE: Peeked [`Kind::Text`] tokens are only one byte long, they may be longer when
/// consumed.
pub fn peek(&mut self) -> Option<&Token> {
if self.next.is_none() {
@ -150,22 +150,21 @@ impl<'s> Lexer<'s> {
fn token(&mut self) -> Option<Token> {
self.len = 0;
let first = self.eat_char()?;
let escape = self.escape;
let kind = match first {
_ if escape && first == '\n' => Hardbreak,
_ if escape
&& matches!(first, '\t' | ' ')
&& self.chars.clone().find(|c| !matches!(c, ' ' | '\t')) == Some('\n') =>
let kind = if self.escape {
self.escape = false;
match self.eat_char()? {
'\n' => Hardbreak,
'\t' | ' '
if self.chars.clone().find(|c| !matches!(c, ' ' | '\t')) == Some('\n') =>
{
while self.eat_char() != Some('\n') {}
Hardbreak
}
_ if escape && first == ' ' => Nbsp,
_ if escape => Text,
' ' => Nbsp,
_ => Text,
}
} else {
match self.eat_char()? {
'\n' => Newline,
'\\' => {
@ -251,11 +250,8 @@ impl<'s> Lexer<'s> {
}
_ => Text,
};
if escape {
self.escape = false;
}
};
Some(Token {
kind,