lex: remove DollarBacktick token

can now read src to check dollar before backticks, so simpler to always
use backtick token

fixes several bugs, e.g.

| $`abc` | (previously did not get closed)

`$abc$` (previously did not get closed)
This commit is contained in:
Noah Hellman 2023-05-10 17:46:00 +02:00
parent b2d15383e7
commit 116fb04725
2 changed files with 34 additions and 36 deletions

View file

@ -372,19 +372,38 @@ impl<'s> Parser<'s> {
self.push(EventKind::Str); self.push(EventKind::Str);
}; };
Some(Continue) Some(Continue)
} else { } else if matches!(first.kind, lex::Kind::Seq(Sequence::Backtick)) {
let (ty, len_opener) = match first.kind { let len_opener = u8::try_from(first.len).ok()?;
lex::Kind::DollarBacktick(l) if first.len - l as usize == 1 => { let ty = if let Some(sp) = self
Some((InlineMath, l)) .events
} .back()
lex::Kind::DollarBacktick(l) if first.len - l as usize == 2 => { .and_then(|e| matches!(&e.kind, EventKind::Str).then(|| e.span))
Some((DisplayMath, l)) .filter(|sp| {
} sp.end() == self.input.span.start()
lex::Kind::Seq(Sequence::Backtick) if first.len < 256 => { && sp.of(self.input.src).as_bytes()[sp.len() - 1] == b'$'
Some((Verbatim, first.len as u8)) && sp
} .end()
_ => None, .checked_sub(2)
}?; .map_or(true, |i| self.input.src.as_bytes()[i] != b'\\')
}) {
let (ty, num_dollar) = if sp.len() > 1
&& sp.of(self.input.src).as_bytes()[sp.len() - 2] == b'$'
&& sp
.end()
.checked_sub(3)
.map_or(true, |i| self.input.src.as_bytes()[i] != b'\\')
{
(DisplayMath, 2)
} else {
(InlineMath, 1)
};
let border = sp.end() - num_dollar;
self.events.back_mut().unwrap().span = Span::new(sp.start(), border);
self.input.span = Span::new(border, self.input.span.end());
ty
} else {
Verbatim
};
self.push_sp(EventKind::Placeholder, self.input.span.empty_before()); self.push_sp(EventKind::Placeholder, self.input.span.empty_before());
self.verbatim = Some(VerbatimState { self.verbatim = Some(VerbatimState {
event_opener: self.events.len(), event_opener: self.events.len(),
@ -394,6 +413,8 @@ impl<'s> Parser<'s> {
}); });
self.attributes = None; self.attributes = None;
self.push(EventKind::Enter(ty)) self.push(EventKind::Enter(ty))
} else {
None
} }
} }

View file

@ -20,7 +20,6 @@ pub enum Kind {
Close(Delimiter), Close(Delimiter),
Sym(Symbol), Sym(Symbol),
Seq(Sequence), Seq(Sequence),
DollarBacktick(u8),
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@ -239,20 +238,6 @@ impl<'s> Lexer<'s> {
'`' => self.eat_seq(Backtick), '`' => self.eat_seq(Backtick),
'.' => self.eat_seq(Period), '.' => self.eat_seq(Period),
'$' => {
self.eat_while(|c| c == '$');
let mut n_ticks: u8 = 0;
self.eat_while(|c| {
if c == '`' {
if let Some(l) = n_ticks.checked_add(1) {
n_ticks = l;
return true;
}
}
false
});
DollarBacktick(n_ticks)
}
_ => Text, _ => Text,
} }
@ -312,7 +297,6 @@ fn is_special(c: char) -> bool {
| ':' | ':'
| '`' | '`'
| '.' | '.'
| '$'
| '\n' | '\n'
) )
} }
@ -418,11 +402,4 @@ mod test {
Seq(Period).l(1), Seq(Period).l(1),
); );
} }
#[test]
fn dollar_backtick() {
test_lex!("$`", DollarBacktick(1).l(2));
test_lex!("$$$`", DollarBacktick(1).l(4));
test_lex!("$$````", DollarBacktick(4).l(6));
}
} }