fnmain/jotdown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fixup! lexer peekable

Browse source
This commit is contained in:
Noah Hellman 2022-12-11 09:37:57 +01:00
commit 72bedb53b4
1 changed files with 17 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/lex.rs
View file

@ -101,11 +101,26 @@ impl<'s> Lexer<'s> {
pub fn peek(&mut self) -> Option<&Token> { pub fn peek(&mut self) -> Option<&Token> {
if self.next.is_none() { if self.next.is_none() {
self.next = self.token(); self.next = self.next_token();
} }
self.next.as_ref() self.next.as_ref()
} }
fn next_token(&mut self) -> Option<Token> {
let mut current = self.token();
// concatenate text tokens
if let Some(Token { kind: Text, len }) = &mut current {
self.next = self.token();
while let Some(Token { kind: Text, len: l }) = self.next {
*len += l;
self.next = self.token();
}
}
current
}
fn peek_char(&mut self) -> char { fn peek_char(&mut self) -> char {
self.chars.clone().next().unwrap_or(EOF) self.chars.clone().next().unwrap_or(EOF)
} }
@ -246,20 +261,7 @@ impl<'s> Iterator for Lexer<'s> {
type Item = Token; type Item = Token;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.next.take().or_else(|| { self.next.take().or_else(|| self.next_token())
let mut current = self.token();
// concatenate text tokens
if let Some(Token { kind: Text, len }) = &mut current {
self.next = self.token();
while let Some(Token { kind: Text, len: l }) = self.next {
*len += l;
self.next = self.token();
}
}
current
})
} }
} }