lex: remove DollarBacktick token

can now read src to check dollar before backticks, so simpler to always
use backtick token

fixes several bugs, e.g.

| $`abc` | (previously did not get closed)

`$abc$` (previously did not get closed)
This commit is contained in:
Noah Hellman 2023-05-10 17:46:00 +02:00
parent b2d15383e7
commit 116fb04725
2 changed files with 34 additions and 36 deletions

View file

@ -372,19 +372,38 @@ impl<'s> Parser<'s> {
self.push(EventKind::Str);
};
Some(Continue)
} else if matches!(first.kind, lex::Kind::Seq(Sequence::Backtick)) {
let len_opener = u8::try_from(first.len).ok()?;
let ty = if let Some(sp) = self
.events
.back()
.and_then(|e| matches!(&e.kind, EventKind::Str).then(|| e.span))
.filter(|sp| {
sp.end() == self.input.span.start()
&& sp.of(self.input.src).as_bytes()[sp.len() - 1] == b'$'
&& sp
.end()
.checked_sub(2)
.map_or(true, |i| self.input.src.as_bytes()[i] != b'\\')
}) {
let (ty, num_dollar) = if sp.len() > 1
&& sp.of(self.input.src).as_bytes()[sp.len() - 2] == b'$'
&& sp
.end()
.checked_sub(3)
.map_or(true, |i| self.input.src.as_bytes()[i] != b'\\')
{
(DisplayMath, 2)
} else {
let (ty, len_opener) = match first.kind {
lex::Kind::DollarBacktick(l) if first.len - l as usize == 1 => {
Some((InlineMath, l))
}
lex::Kind::DollarBacktick(l) if first.len - l as usize == 2 => {
Some((DisplayMath, l))
}
lex::Kind::Seq(Sequence::Backtick) if first.len < 256 => {
Some((Verbatim, first.len as u8))
}
_ => None,
}?;
(InlineMath, 1)
};
let border = sp.end() - num_dollar;
self.events.back_mut().unwrap().span = Span::new(sp.start(), border);
self.input.span = Span::new(border, self.input.span.end());
ty
} else {
Verbatim
};
self.push_sp(EventKind::Placeholder, self.input.span.empty_before());
self.verbatim = Some(VerbatimState {
event_opener: self.events.len(),
@ -394,6 +413,8 @@ impl<'s> Parser<'s> {
});
self.attributes = None;
self.push(EventKind::Enter(ty))
} else {
None
}
}

View file

@ -20,7 +20,6 @@ pub enum Kind {
Close(Delimiter),
Sym(Symbol),
Seq(Sequence),
DollarBacktick(u8),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@ -239,20 +238,6 @@ impl<'s> Lexer<'s> {
'`' => self.eat_seq(Backtick),
'.' => self.eat_seq(Period),
'$' => {
self.eat_while(|c| c == '$');
let mut n_ticks: u8 = 0;
self.eat_while(|c| {
if c == '`' {
if let Some(l) = n_ticks.checked_add(1) {
n_ticks = l;
return true;
}
}
false
});
DollarBacktick(n_ticks)
}
_ => Text,
}
@ -312,7 +297,6 @@ fn is_special(c: char) -> bool {
| ':'
| '`'
| '.'
| '$'
| '\n'
)
}
@ -418,11 +402,4 @@ mod test {
Seq(Period).l(1),
);
}
#[test]
fn dollar_backtick() {
test_lex!("$`", DollarBacktick(1).l(2));
test_lex!("$$$`", DollarBacktick(1).l(4));
test_lex!("$$````", DollarBacktick(4).l(6));
}
}