fnmain/jotdown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

lex: separate escaped/non-escaped

Browse source
This commit is contained in:
Noah Hellman 2023-04-22 19:01:26 +02:00
parent e877fdbde8
commit 3701d282ac
1 changed files with 102 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/lex.rs
View file

@ -93,7 +93,7 @@ impl<'s> Lexer<'s> {
} }
} }
/// NOTE: Peeked [`Kind::Text`] tokens are only one char long, they may be longer when /// NOTE: Peeked [`Kind::Text`] tokens are only one byte long, they may be longer when
/// consumed. /// consumed.
pub fn peek(&mut self) -> Option<&Token> { pub fn peek(&mut self) -> Option<&Token> {
if self.next.is_none() { if self.next.is_none() {
@ -150,22 +150,21 @@ impl<'s> Lexer<'s> {
fn token(&mut self) -> Option<Token> { fn token(&mut self) -> Option<Token> {
self.len = 0; self.len = 0;
let first = self.eat_char()?; let kind = if self.escape {
self.escape = false;
let escape = self.escape; match self.eat_char()? {
'\n' => Hardbreak,
let kind = match first { '\t' | ' '
_ if escape && first == '\n' => Hardbreak, if self.chars.clone().find(|c| !matches!(c, ' ' | '\t')) == Some('\n') =>
_ if escape
&& matches!(first, '\t' | ' ')
&& self.chars.clone().find(|c| !matches!(c, ' ' | '\t')) == Some('\n') =>
{ {
while self.eat_char() != Some('\n') {} while self.eat_char() != Some('\n') {}
Hardbreak Hardbreak
} }
_ if escape && first == ' ' => Nbsp, ' ' => Nbsp,
_ if escape => Text, _ => Text,
}
} else {
match self.eat_char()? {
'\n' => Newline, '\n' => Newline,
'\\' => { '\\' => {
@ -251,11 +250,8 @@ impl<'s> Lexer<'s> {
} }
_ => Text, _ => Text,
};
if escape {
self.escape = false;
} }
};
Some(Token { Some(Token {
kind, kind,