116fb04725
can now read src to check dollar before backticks, so simpler to always use backtick token fixes several bugs, e.g. | $`abc` | (previously did not get closed) `$abc$` (previously did not get closed)
405 lines
10 KiB
Rust
405 lines
10 KiB
Rust
use Delimiter::*;
|
|
use Kind::*;
|
|
use Sequence::*;
|
|
use Symbol::*;
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub(crate) struct Token {
|
|
pub kind: Kind,
|
|
pub len: usize,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Kind {
|
|
Text,
|
|
Newline,
|
|
Nbsp,
|
|
Hardbreak,
|
|
Escape,
|
|
Open(Delimiter),
|
|
Close(Delimiter),
|
|
Sym(Symbol),
|
|
Seq(Sequence),
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Delimiter {
|
|
Brace,
|
|
BraceAsterisk,
|
|
BraceCaret,
|
|
BraceEqual,
|
|
BraceHyphen,
|
|
BracePlus,
|
|
BraceTilde,
|
|
BraceUnderscore,
|
|
Bracket,
|
|
BraceQuote1,
|
|
BraceQuote2,
|
|
Paren,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Symbol {
|
|
Asterisk,
|
|
Caret,
|
|
ExclaimBracket,
|
|
Lt,
|
|
Pipe,
|
|
Quote1,
|
|
Quote2,
|
|
Tilde,
|
|
Underscore,
|
|
Colon,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Sequence {
|
|
Backtick,
|
|
Hyphen,
|
|
Period,
|
|
}
|
|
|
|
impl Sequence {
|
|
fn ch(self) -> char {
|
|
match self {
|
|
Self::Backtick => '`',
|
|
Self::Period => '.',
|
|
Self::Hyphen => '-',
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub(crate) struct Lexer<'s> {
|
|
src: &'s str,
|
|
chars: std::str::Chars<'s>,
|
|
/// Next character should be escaped.
|
|
escape: bool,
|
|
/// Token to be peeked or next'ed.
|
|
next: Option<Token>,
|
|
/// Length of current token.
|
|
len: usize,
|
|
}
|
|
|
|
impl<'s> Lexer<'s> {
|
|
pub fn new(src: &'s str) -> Self {
|
|
Lexer {
|
|
src,
|
|
chars: src.chars(),
|
|
escape: false,
|
|
next: None,
|
|
len: 0,
|
|
}
|
|
}
|
|
|
|
/// NOTE: Peeked [`Kind::Text`] tokens are only one byte long, they may be longer when
|
|
/// consumed.
|
|
pub fn peek(&mut self) -> Option<&Token> {
|
|
if self.next.is_none() {
|
|
self.next = self.token();
|
|
}
|
|
self.next.as_ref()
|
|
}
|
|
|
|
pub fn ahead(&self) -> &'s str {
|
|
let pos =
|
|
self.src.len() - self.chars.as_str().len() - self.next.as_ref().map_or(0, |t| t.len);
|
|
&self.src[pos..]
|
|
}
|
|
|
|
fn next_token(&mut self) -> Option<Token> {
|
|
let mut current = self.token();
|
|
|
|
// concatenate text tokens
|
|
if let Some(Token { kind: Text, len }) = &mut current {
|
|
self.next = self.token();
|
|
while let Some(Token { kind: Text, len: l }) = self.next {
|
|
*len += l;
|
|
self.next = self.token();
|
|
}
|
|
}
|
|
|
|
current
|
|
}
|
|
|
|
fn peek_char_n(&mut self, n: usize) -> Option<char> {
|
|
self.chars.clone().nth(n)
|
|
}
|
|
|
|
fn peek_char(&mut self) -> Option<char> {
|
|
self.peek_char_n(0)
|
|
}
|
|
|
|
fn eat_char(&mut self) -> Option<char> {
|
|
let c = self.chars.next();
|
|
self.len += c.map_or(0, char::len_utf8);
|
|
c
|
|
}
|
|
|
|
fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
|
|
while let Some(c) = self.peek_char() {
|
|
if predicate(c) {
|
|
self.eat_char();
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
fn token(&mut self) -> Option<Token> {
|
|
self.len = 0;
|
|
|
|
let kind = if self.escape {
|
|
self.escape = false;
|
|
match self.eat_char()? {
|
|
'\n' => Hardbreak,
|
|
'\t' | ' '
|
|
if self.chars.clone().find(|c| !matches!(c, ' ' | '\t')) == Some('\n') =>
|
|
{
|
|
while self.eat_char() != Some('\n') {}
|
|
Hardbreak
|
|
}
|
|
' ' => Nbsp,
|
|
_ => Text,
|
|
}
|
|
} else {
|
|
self.eat_while(|c| !is_special(c));
|
|
if self.len > 0 {
|
|
Text
|
|
} else {
|
|
match self.eat_char()? {
|
|
'\n' => Newline,
|
|
|
|
'\\' => {
|
|
if self
|
|
.peek_char()
|
|
.map_or(false, |c| c.is_whitespace() || c.is_ascii_punctuation())
|
|
{
|
|
self.escape = true;
|
|
Escape
|
|
} else {
|
|
Text
|
|
}
|
|
}
|
|
|
|
'[' => Open(Bracket),
|
|
']' => Close(Bracket),
|
|
'(' => Open(Paren),
|
|
')' => Close(Paren),
|
|
'{' => {
|
|
let explicit = match self.peek_char() {
|
|
Some('*') => Some(Open(BraceAsterisk)),
|
|
Some('^') => Some(Open(BraceCaret)),
|
|
Some('=') => Some(Open(BraceEqual)),
|
|
Some('-') => Some(Open(BraceHyphen)),
|
|
Some('+') => Some(Open(BracePlus)),
|
|
Some('~') => Some(Open(BraceTilde)),
|
|
Some('_') => Some(Open(BraceUnderscore)),
|
|
Some('\'') => Some(Open(BraceQuote1)),
|
|
Some('"') => Some(Open(BraceQuote2)),
|
|
_ => None,
|
|
};
|
|
if let Some(exp) = explicit {
|
|
self.eat_char();
|
|
exp
|
|
} else {
|
|
Open(Brace)
|
|
}
|
|
}
|
|
'}' => Close(Brace),
|
|
'*' => self.maybe_eat_close_brace(Sym(Asterisk), BraceAsterisk),
|
|
'^' => self.maybe_eat_close_brace(Sym(Caret), BraceCaret),
|
|
'=' => self.maybe_eat_close_brace(Text, BraceEqual),
|
|
'+' => self.maybe_eat_close_brace(Text, BracePlus),
|
|
'~' => self.maybe_eat_close_brace(Sym(Tilde), BraceTilde),
|
|
'_' => self.maybe_eat_close_brace(Sym(Underscore), BraceUnderscore),
|
|
'\'' => self.maybe_eat_close_brace(Sym(Quote1), BraceQuote1),
|
|
'"' => self.maybe_eat_close_brace(Sym(Quote2), BraceQuote2),
|
|
'-' => {
|
|
if self.peek_char() == Some('}') {
|
|
self.eat_char();
|
|
Close(BraceHyphen)
|
|
} else {
|
|
while self.peek_char() == Some('-') && self.peek_char_n(1) != Some('}')
|
|
{
|
|
self.eat_char();
|
|
}
|
|
Seq(Hyphen)
|
|
}
|
|
}
|
|
|
|
'!' if self.peek_char() == Some('[') => {
|
|
self.eat_char();
|
|
Sym(ExclaimBracket)
|
|
}
|
|
'<' => Sym(Lt),
|
|
'|' => Sym(Pipe),
|
|
':' => Sym(Colon),
|
|
|
|
'`' => self.eat_seq(Backtick),
|
|
'.' => self.eat_seq(Period),
|
|
|
|
_ => Text,
|
|
}
|
|
}
|
|
};
|
|
|
|
Some(Token {
|
|
kind,
|
|
len: self.len,
|
|
})
|
|
}
|
|
|
|
fn eat_seq(&mut self, s: Sequence) -> Kind {
|
|
self.eat_while(|c| c == s.ch());
|
|
Seq(s)
|
|
}
|
|
|
|
fn maybe_eat_close_brace(&mut self, kind: Kind, d: Delimiter) -> Kind {
|
|
if self.peek_char() == Some('}') {
|
|
self.eat_char();
|
|
Close(d)
|
|
} else {
|
|
kind
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<'s> Iterator for Lexer<'s> {
|
|
type Item = Token;
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
self.next.take().or_else(|| self.next_token())
|
|
}
|
|
}
|
|
|
|
fn is_special(c: char) -> bool {
|
|
matches!(
|
|
c,
|
|
'\\' | '['
|
|
| ']'
|
|
| '('
|
|
| ')'
|
|
| '{'
|
|
| '}'
|
|
| '*'
|
|
| '^'
|
|
| '='
|
|
| '+'
|
|
| '~'
|
|
| '_'
|
|
| '\''
|
|
| '"'
|
|
| '-'
|
|
| '!'
|
|
| '<'
|
|
| '|'
|
|
| ':'
|
|
| '`'
|
|
| '.'
|
|
| '\n'
|
|
)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod test {
|
|
use super::Delimiter::*;
|
|
use super::Kind::*;
|
|
use super::Sequence::*;
|
|
use super::Symbol::*;
|
|
|
|
macro_rules! test_lex {
|
|
($($st:ident,)? $src:expr $(,$($token:expr),* $(,)?)?) => {
|
|
#[allow(unused)]
|
|
let actual = super::Lexer::new($src).collect::<Vec<_>>();
|
|
let expected = vec![$($($token),*,)?];
|
|
assert_eq!(actual, expected, "{}", $src);
|
|
};
|
|
}
|
|
|
|
impl super::Kind {
|
|
fn l(self, len: usize) -> super::Token {
|
|
super::Token { kind: self, len }
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn empty() {
|
|
test_lex!("");
|
|
}
|
|
|
|
#[test]
|
|
fn basic() {
|
|
test_lex!("abc", Text.l(3));
|
|
test_lex!(
|
|
"para w/ some _emphasis_ and *strong*.",
|
|
Text.l(13),
|
|
Sym(Underscore).l(1),
|
|
Text.l(8),
|
|
Sym(Underscore).l(1),
|
|
Text.l(5),
|
|
Sym(Asterisk).l(1),
|
|
Text.l(6),
|
|
Sym(Asterisk).l(1),
|
|
Seq(Period).l(1),
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn escape() {
|
|
test_lex!(r#"\a"#, Text.l(2));
|
|
test_lex!(r#"\\a"#, Escape.l(1), Text.l(2));
|
|
test_lex!(r#"\."#, Escape.l(1), Text.l(1));
|
|
test_lex!(r#"\ "#, Escape.l(1), Nbsp.l(1));
|
|
test_lex!(r#"\{-"#, Escape.l(1), Text.l(1), Seq(Hyphen).l(1));
|
|
}
|
|
|
|
#[test]
|
|
fn hardbreak() {
|
|
test_lex!("a\\\n", Text.l(1), Escape.l(1), Hardbreak.l(1));
|
|
test_lex!("a\\ \n", Text.l(1), Escape.l(1), Hardbreak.l(4));
|
|
test_lex!("a\\\t \t \n", Text.l(1), Escape.l(1), Hardbreak.l(5));
|
|
}
|
|
|
|
#[test]
|
|
fn delim() {
|
|
test_lex!("{-", Open(BraceHyphen).l(2));
|
|
test_lex!("-}", Close(BraceHyphen).l(2));
|
|
test_lex!("{++}", Open(BracePlus).l(2), Close(BracePlus).l(2));
|
|
}
|
|
|
|
#[test]
|
|
fn sym() {
|
|
test_lex!(
|
|
r#"'*^![<|"~_"#,
|
|
Sym(Quote1).l(1),
|
|
Sym(Asterisk).l(1),
|
|
Sym(Caret).l(1),
|
|
Sym(ExclaimBracket).l(2),
|
|
Sym(Lt).l(1),
|
|
Sym(Pipe).l(1),
|
|
Sym(Quote2).l(1),
|
|
Sym(Tilde).l(1),
|
|
Sym(Underscore).l(1),
|
|
);
|
|
test_lex!(
|
|
"''''",
|
|
Sym(Quote1).l(1),
|
|
Sym(Quote1).l(1),
|
|
Sym(Quote1).l(1),
|
|
Sym(Quote1).l(1),
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn seq() {
|
|
test_lex!("`", Seq(Backtick).l(1));
|
|
test_lex!("```", Seq(Backtick).l(3));
|
|
test_lex!(
|
|
"`-.",
|
|
Seq(Backtick).l(1),
|
|
Seq(Hyphen).l(1),
|
|
Seq(Period).l(1),
|
|
);
|
|
}
|
|
}
|