inline: impl verbatim w/o lookahead

avoid lookahead for containers that can cross newline

needed to get rid of DiscontinuousChars iter
This commit is contained in:
Noah Hellman 2023-02-15 21:18:08 +01:00
parent e8e551fd8b
commit ed5aed3759
3 changed files with 195 additions and 148 deletions

View file

@ -127,6 +127,50 @@ impl<I: Iterator<Item = char> + Clone> Input<I> {
None None
} }
} }
fn ahead_raw_format(&mut self) -> Option<Span> {
if matches!(
self.lexer.peek().map(|t| &t.kind),
Some(lex::Kind::Open(Delimiter::BraceEqual))
) {
let mut ahead = self.lexer.chars();
let mut end = false;
let len = (&mut ahead)
.skip(2) // {=
.take_while(|c| {
if *c == '{' {
return false;
}
if *c == '}' {
end = true;
};
!end && !c.is_whitespace()
})
.map(char::len_utf8)
.sum();
(len > 0 && end).then(|| {
let tok = self.eat();
debug_assert_eq!(
tok,
Some(lex::Token {
kind: lex::Kind::Open(Delimiter::BraceEqual),
len: 2,
})
);
self.lexer = lex::Lexer::new(ahead);
self.span.after(len)
})
} else {
None
}
}
}
pub struct VerbatimState {
event_opener: usize,
len_opener: u8,
non_whitespace_encountered: bool,
non_whitespace_last: Option<(lex::Kind, usize)>,
} }
pub struct Parser<I: Iterator + Clone> { pub struct Parser<I: Iterator + Clone> {
@ -136,6 +180,8 @@ pub struct Parser<I: Iterator + Clone> {
/// Buffer queue for next events. Events are buffered until no modifications due to future /// Buffer queue for next events. Events are buffered until no modifications due to future
/// characters are needed. /// characters are needed.
events: std::collections::VecDeque<Event>, events: std::collections::VecDeque<Event>,
/// State if inside a verbatim container.
verbatim: Option<VerbatimState>,
} }
impl<I: Iterator<Item = char> + Clone> Parser<I> { impl<I: Iterator<Item = char> + Clone> Parser<I> {
@ -144,6 +190,7 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
input: Input::new(chars), input: Input::new(chars),
openers: Vec::new(), openers: Vec::new(),
events: std::collections::VecDeque::new(), events: std::collections::VecDeque::new(),
verbatim: None,
} }
} }
@ -151,6 +198,7 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
self.input.reset(chars); self.input.reset(chars);
self.openers.clear(); self.openers.clear();
debug_assert!(self.events.is_empty()); debug_assert!(self.events.is_empty());
debug_assert!(self.verbatim.is_none());
} }
fn push_sp(&mut self, kind: EventKind, span: Span) -> Option<()> { fn push_sp(&mut self, kind: EventKind, span: Span) -> Option<()> {
@ -183,125 +231,92 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
} }
fn parse_verbatim(&mut self, first: &lex::Token) -> Option<()> { fn parse_verbatim(&mut self, first: &lex::Token) -> Option<()> {
let (mut kind, opener_len) = match first.kind { if let Some(VerbatimState {
lex::Kind::Seq(Sequence::Dollar) => { event_opener,
let math_opt = (first.len <= 2) len_opener,
.then(|| { non_whitespace_encountered,
if let Some(lex::Token { non_whitespace_last,
kind: lex::Kind::Seq(Sequence::Backtick), }) = &mut self.verbatim
len,
}) = self.input.peek()
{ {
Some(( let event_opener = *event_opener;
if first.len == 2 { let len_opener = *len_opener;
DisplayMath if usize::from(len_opener) == first.len
} else { && matches!(first.kind, lex::Kind::Seq(Sequence::Backtick))
InlineMath
},
*len,
))
} else {
None
}
})
.flatten();
if math_opt.is_some() {
self.input.eat(); // backticks
}
math_opt
}
lex::Kind::Seq(Sequence::Backtick) => Some((Verbatim, first.len)),
_ => None,
}?;
let e_attr = self.events.len();
self.push_sp(
EventKind::Placeholder,
Span::empty_at(self.input.span.start()),
);
let opener_event = self.events.len();
self.push(EventKind::Enter(kind));
let mut span_inner = self.input.span.empty_after();
let mut span_outer = None;
let mut non_whitespace_first = None;
let mut non_whitespace_last = None;
while let Some(t) = self.input.eat() {
if matches!(t.kind, lex::Kind::Seq(Sequence::Backtick)) && t.len == opener_len {
if matches!(kind, Verbatim)
&& matches!(
self.input.peek().map(|t| &t.kind),
Some(lex::Kind::Open(Delimiter::BraceEqual))
)
{ {
let mut ahead = self.input.lexer.chars(); let raw_format = self.input.ahead_raw_format();
let mut end = false; let mut span_closer = self.input.span;
let len = (&mut ahead) if let Some(span_format) = raw_format {
.skip(2) // {= self.events[event_opener].kind = EventKind::Enter(RawFormat);
.take_while(|c| { self.events[event_opener].span = span_format;
if *c == '{' { self.input.span = span_format.translate(1);
return false; span_closer = span_format;
} } else if let Some((non_empty, span_attr)) = self.input.ahead_attributes() {
if *c == '}' {
end = true;
};
!end && !c.is_whitespace()
})
.map(char::len_utf8)
.sum();
if len > 0 && end {
let tok = self.input.eat();
debug_assert_eq!(
tok,
Some(lex::Token {
kind: lex::Kind::Open(Delimiter::BraceEqual),
len: 2,
})
);
self.input.lexer = lex::Lexer::new(ahead);
let span_format = self.input.span.after(len);
kind = RawFormat;
self.events[opener_event].kind = EventKind::Enter(kind);
self.events[opener_event].span = span_format;
self.input.span = span_format.translate(1); // }
span_outer = Some(span_format);
}
}
break;
}
if !matches!(t.kind, lex::Kind::Whitespace) {
if non_whitespace_first.is_none() {
non_whitespace_first = Some((t.kind, span_inner.end()));
}
non_whitespace_last = Some((t.kind, span_inner.end() + t.len));
}
span_inner = span_inner.extend(t.len);
self.input.reset_span();
}
if let Some((lex::Kind::Seq(Sequence::Backtick), pos)) = non_whitespace_first {
span_inner = span_inner.with_start(pos);
}
if let Some((lex::Kind::Seq(Sequence::Backtick), pos)) = non_whitespace_last {
span_inner = span_inner.with_end(pos);
}
self.push_sp(EventKind::Str, span_inner);
self.push_sp(EventKind::Exit(kind), span_outer.unwrap_or(self.input.span));
if let Some((non_empty, span)) = self.input.ahead_attributes() {
self.input.span = span;
if non_empty { if non_empty {
let e_attr = event_opener - 1;
self.events[e_attr] = Event { self.events[e_attr] = Event {
kind: EventKind::Attributes, kind: EventKind::Attributes,
span, span: span_attr,
}; };
} }
self.input.span = span_attr;
};
let ty_opener = if let EventKind::Enter(ty) = self.events[event_opener].kind {
debug_assert!(matches!(
ty,
Verbatim | RawFormat | InlineMath | DisplayMath
));
ty
} else {
panic!()
};
if let Some((lex::Kind::Seq(Sequence::Backtick), event_skip)) = non_whitespace_last
{
self.events.drain(*event_skip..);
} }
self.push_sp(EventKind::Exit(ty_opener), span_closer);
self.verbatim = None;
} else {
// continue verbatim
if matches!(first.kind, lex::Kind::Whitespace) {
if !*non_whitespace_encountered
&& self.input.peek().map_or(false, |t| {
matches!(
t.kind,
lex::Kind::Seq(Sequence::Backtick) if t.len != len_opener.into(),
)
})
{
return Some(()); // skip whitespace
}
} else {
*non_whitespace_encountered = true;
*non_whitespace_last = Some((first.kind, self.events.len() + 1));
}
self.push(EventKind::Str);
};
Some(()) Some(())
} else {
let (ty, len_opener) = match first.kind {
lex::Kind::DollarBacktick(l) if first.len - l as usize == 1 => {
Some((InlineMath, l))
}
lex::Kind::DollarBacktick(l) if first.len - l as usize == 2 => {
Some((DisplayMath, l))
}
lex::Kind::Seq(Sequence::Backtick) if first.len < 256 => {
Some((Verbatim, first.len as u8))
}
_ => None,
}?;
self.push_sp(EventKind::Placeholder, self.input.span.empty_before());
self.verbatim = Some(VerbatimState {
event_opener: self.events.len(),
len_opener,
non_whitespace_encountered: false,
non_whitespace_last: None,
});
self.push(EventKind::Enter(ty))
}
} }
fn parse_attributes(&mut self, first: &lex::Token) -> Option<()> { fn parse_attributes(&mut self, first: &lex::Token) -> Option<()> {
@ -339,10 +354,11 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
self.push(EventKind::Attributes); self.push(EventKind::Attributes);
self.push_sp(EventKind::Enter(Span), span_str.empty_before()); self.push_sp(EventKind::Enter(Span), span_str.empty_before());
self.push_sp(EventKind::Str, span_str); self.push_sp(EventKind::Str, span_str);
return self.push_sp(EventKind::Exit(Span), span_str.empty_after()); self.push_sp(EventKind::Exit(Span), span_str.empty_after());
} else { } else {
return self.push_sp(EventKind::Placeholder, self.input.span.empty_before()); self.push_sp(EventKind::Placeholder, self.input.span.empty_before());
} }
return Some(());
} }
} }
@ -400,9 +416,9 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
if end && valid { if end && valid {
self.input.lexer = lex::Lexer::new(ahead); self.input.lexer = lex::Lexer::new(ahead);
self.input.span = self.input.span.after(len); self.input.span = self.input.span.after(len);
let span = self.input.span; self.push(EventKind::Atom(Symbol));
self.input.span = self.input.span.after(1); self.input.span = self.input.span.after(1);
return self.push_sp(EventKind::Atom(Symbol), span); return Some(());
} }
} }
None None
@ -452,13 +468,11 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
} }
fn parse_container(&mut self, first: &lex::Token) -> Option<()> { fn parse_container(&mut self, first: &lex::Token) -> Option<()> {
let (delim, dir) = Delim::from_token(first.kind)?; let (delim, dir) = Delim::from_token(first)?;
self.openers self.openers
.iter() .iter()
.rposition(|(d, _)| { .rposition(|(d, _)| delim.closes(*d))
*d == delim || matches!((d, delim), (Delim::Span(..), Delim::Span(..)))
})
.and_then(|o| { .and_then(|o| {
if matches!(dir, Dir::Open) { if matches!(dir, Dir::Open) {
return None; return None;
@ -479,6 +493,7 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
} }
let inner_span = self.events[e_opener].span.between(self.input.span); let inner_span = self.events[e_opener].span.between(self.input.span);
self.openers.drain(o..);
let mut closed = match DelimEventKind::from(d) { let mut closed = match DelimEventKind::from(d) {
DelimEventKind::Container(cont) => { DelimEventKind::Container(cont) => {
self.events[e_opener].kind = EventKind::Enter(cont); self.events[e_opener].kind = EventKind::Enter(cont);
@ -490,7 +505,6 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
} }
DelimEventKind::Span(ty) => self.post_span(ty, e_opener), DelimEventKind::Span(ty) => self.post_span(ty, e_opener),
}; };
self.openers.drain(o..);
if closed.is_some() { if closed.is_some() {
let event_closer = &mut self.events.back_mut().unwrap(); let event_closer = &mut self.events.back_mut().unwrap();
@ -566,7 +580,7 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
fn post_span(&mut self, ty: SpanType, opener_event: usize) -> Option<()> { fn post_span(&mut self, ty: SpanType, opener_event: usize) -> Option<()> {
let mut ahead = self.input.lexer.chars(); let mut ahead = self.input.lexer.chars();
let (kind, span) = match ahead.next() { let kind = match ahead.next() {
Some(opener @ ('[' | '(')) => { Some(opener @ ('[' | '(')) => {
let img = ty == SpanType::Image; let img = ty == SpanType::Image;
let (closer, kind) = match opener { let (closer, kind) = match opener {
@ -587,19 +601,22 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
}) })
.map(char::len_utf8) .map(char::len_utf8)
.sum(); .sum();
end.then(|| { if end {
let span = self.input.span.after(len).translate(1); self.input.span = self.input.span.after(len).translate(1);
(kind, span) Some(kind)
}) } else {
None
}
} }
_ => None, _ => None,
}?; }?;
self.input.lexer = lex::Lexer::new(ahead); self.input.lexer = lex::Lexer::new(ahead);
self.events[opener_event].kind = EventKind::Enter(kind); self.events[opener_event].kind = EventKind::Enter(kind);
self.events[opener_event].span = span; self.events[opener_event].span = self.input.span;
self.input.span = span.translate(1); self.push(EventKind::Exit(kind));
self.push_sp(EventKind::Exit(kind), span) self.input.span = self.input.span.translate(1);
Some(())
} }
fn parse_atom(&mut self, first: &lex::Token) -> Option<()> { fn parse_atom(&mut self, first: &lex::Token) -> Option<()> {
@ -695,13 +712,13 @@ enum Dir {
} }
impl Delim { impl Delim {
fn from_token(kind: lex::Kind) -> Option<(Self, Dir)> { fn from_token(token: &lex::Token) -> Option<(Self, Dir)> {
use Delim::*; use Delim::*;
use Dir::{Both, Close, Open}; use Dir::{Both, Close, Open};
use Directionality::{Bi, Uni}; use Directionality::{Bi, Uni};
use SpanType::{General, Image}; use SpanType::{General, Image};
match kind { match token.kind {
lex::Kind::Sym(Symbol::Asterisk) => Some((Strong(Bi), Both)), lex::Kind::Sym(Symbol::Asterisk) => Some((Strong(Bi), Both)),
lex::Kind::Sym(Symbol::Underscore) => Some((Emphasis(Bi), Both)), lex::Kind::Sym(Symbol::Underscore) => Some((Emphasis(Bi), Both)),
lex::Kind::Sym(Symbol::Caret) => Some((Superscript(Bi), Both)), lex::Kind::Sym(Symbol::Caret) => Some((Superscript(Bi), Both)),
@ -732,6 +749,10 @@ impl Delim {
_ => None, _ => None,
} }
} }
fn closes(self, opener: Delim) -> bool {
self == opener || matches!((opener, self), (Delim::Span(..), Delim::Span(..)))
}
} }
enum DelimEventKind { enum DelimEventKind {
@ -763,6 +784,7 @@ impl<I: Iterator<Item = char> + Clone> Iterator for Parser<I> {
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
while self.events.is_empty() while self.events.is_empty()
|| !self.openers.is_empty() || !self.openers.is_empty()
|| self.verbatim.is_some()
|| self // for merge or attributes || self // for merge or attributes
.events .events
.back() .back()
@ -775,6 +797,20 @@ impl<I: Iterator<Item = char> + Clone> Iterator for Parser<I> {
} }
} }
// automatically close unclosed verbatim
if let Some(VerbatimState { event_opener, .. }) = self.verbatim.take() {
let ty_opener = if let EventKind::Enter(ty) = self.events[event_opener].kind {
debug_assert!(matches!(
ty,
Verbatim | RawFormat | InlineMath | DisplayMath
));
ty
} else {
panic!()
};
self.push(EventKind::Exit(ty_opener));
}
self.events.pop_front().and_then(|e| { self.events.pop_front().and_then(|e| {
match e.kind { match e.kind {
EventKind::Str if e.span.is_empty() => self.next(), EventKind::Str if e.span.is_empty() => self.next(),

View file

@ -21,6 +21,7 @@ pub enum Kind {
Close(Delimiter), Close(Delimiter),
Sym(Symbol), Sym(Symbol),
Seq(Sequence), Seq(Sequence),
DollarBacktick(u8),
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@ -55,7 +56,6 @@ pub enum Symbol {
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Sequence { pub enum Sequence {
Backtick, Backtick,
Dollar,
Hyphen, Hyphen,
Period, Period,
} }
@ -64,7 +64,6 @@ impl Sequence {
fn ch(self) -> char { fn ch(self) -> char {
match self { match self {
Self::Backtick => '`', Self::Backtick => '`',
Self::Dollar => '$',
Self::Period => '.', Self::Period => '.',
Self::Hyphen => '-', Self::Hyphen => '-',
} }
@ -207,6 +206,7 @@ impl<I: Iterator<Item = char> + Clone> Lexer<I> {
Open(Brace) Open(Brace)
} }
} }
'}' => Close(Brace),
'*' => self.maybe_eat_close_brace(Sym(Asterisk), BraceAsterisk), '*' => self.maybe_eat_close_brace(Sym(Asterisk), BraceAsterisk),
'^' => self.maybe_eat_close_brace(Sym(Caret), BraceCaret), '^' => self.maybe_eat_close_brace(Sym(Caret), BraceCaret),
'=' => self.maybe_eat_close_brace(Text, BraceEqual), '=' => self.maybe_eat_close_brace(Text, BraceEqual),
@ -236,8 +236,21 @@ impl<I: Iterator<Item = char> + Clone> Lexer<I> {
':' => Sym(Colon), ':' => Sym(Colon),
'`' => self.eat_seq(Backtick), '`' => self.eat_seq(Backtick),
'$' => self.eat_seq(Dollar),
'.' => self.eat_seq(Period), '.' => self.eat_seq(Period),
'$' => {
self.eat_while(|c| c == '$');
let mut n_ticks: u8 = 0;
self.eat_while(|c| {
if c == '`' {
if let Some(l) = n_ticks.checked_add(1) {
n_ticks = l;
return true;
}
}
false
});
DollarBacktick(n_ticks)
}
_ => Text, _ => Text,
}; };
@ -383,11 +396,17 @@ mod test {
test_lex!("`", Seq(Backtick).l(1)); test_lex!("`", Seq(Backtick).l(1));
test_lex!("```", Seq(Backtick).l(3)); test_lex!("```", Seq(Backtick).l(3));
test_lex!( test_lex!(
"`$-.", "`-.",
Seq(Backtick).l(1), Seq(Backtick).l(1),
Seq(Dollar).l(1),
Seq(Hyphen).l(1), Seq(Hyphen).l(1),
Seq(Period).l(1), Seq(Period).l(1),
); );
} }
#[test]
fn dollar_backtick() {
test_lex!("$`", DollarBacktick(1).l(2));
test_lex!("$$$`", DollarBacktick(1).l(4));
test_lex!("$$````", DollarBacktick(4).l(6));
}
} }

View file

@ -30,14 +30,6 @@ impl Span {
Self::empty_at(self.end()) Self::empty_at(self.end())
} }
pub fn with_start(self, start: usize) -> Self {
Self::new(start, self.end())
}
pub fn with_end(self, end: usize) -> Self {
Self::new(self.start(), end)
}
pub fn with_len(self, len: usize) -> Self { pub fn with_len(self, len: usize) -> Self {
Self::by_len(self.start(), len) Self::by_len(self.start(), len)
} }