inline: impl verbatim w/o lookahead

avoid lookahead for containers that can cross newline

needed to get rid of DiscontinuousChars iter

src/inline.rs

@@ -127,6 +127,50 @@ impl<I: Iterator<Item = char> + Clone> Input<I> {
             None
         }
     }
+
+    fn ahead_raw_format(&mut self) -> Option<Span> {
+        if matches!(
+            self.lexer.peek().map(|t| &t.kind),
+            Some(lex::Kind::Open(Delimiter::BraceEqual))
+        ) {
+            let mut ahead = self.lexer.chars();
+            let mut end = false;
+            let len = (&mut ahead)
+                .skip(2) // {=
+                .take_while(|c| {
+                    if *c == '{' {
+                        return false;
+                    }
+                    if *c == '}' {
+                        end = true;
+                    };
+                    !end && !c.is_whitespace()
+                })
+                .map(char::len_utf8)
+                .sum();
+            (len > 0 && end).then(|| {
+                let tok = self.eat();
+                debug_assert_eq!(
+                    tok,
+                    Some(lex::Token {
+                        kind: lex::Kind::Open(Delimiter::BraceEqual),
+                        len: 2,
+                    })
+                );
+                self.lexer = lex::Lexer::new(ahead);
+                self.span.after(len)
+            })
+        } else {
+            None
+        }
+    }
+}
+
+pub struct VerbatimState {
+    event_opener: usize,
+    len_opener: u8,
+    non_whitespace_encountered: bool,
+    non_whitespace_last: Option<(lex::Kind, usize)>,
 }
 
 pub struct Parser<I: Iterator + Clone> {
@@ -136,6 +180,8 @@ pub struct Parser<I: Iterator + Clone> {
     /// Buffer queue for next events. Events are buffered until no modifications due to future
     /// characters are needed.
     events: std::collections::VecDeque<Event>,
+    /// State if inside a verbatim container.
+    verbatim: Option<VerbatimState>,
 }
 
 impl<I: Iterator<Item = char> + Clone> Parser<I> {
@@ -144,6 +190,7 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
             input: Input::new(chars),
             openers: Vec::new(),
             events: std::collections::VecDeque::new(),
+            verbatim: None,
         }
     }
 
@@ -151,6 +198,7 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
         self.input.reset(chars);
         self.openers.clear();
         debug_assert!(self.events.is_empty());
+        debug_assert!(self.verbatim.is_none());
     }
 
     fn push_sp(&mut self, kind: EventKind, span: Span) -> Option<()> {
@@ -183,125 +231,92 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
     }
 
     fn parse_verbatim(&mut self, first: &lex::Token) -> Option<()> {
-        let (mut kind, opener_len) = match first.kind {
+        if let Some(VerbatimState {
-            lex::Kind::Seq(Sequence::Dollar) => {
+            event_opener,
-                let math_opt = (first.len <= 2)
+            len_opener,
-                    .then(|| {
+            non_whitespace_encountered,
-                        if let Some(lex::Token {
+            non_whitespace_last,
-                            kind: lex::Kind::Seq(Sequence::Backtick),
+        }) = &mut self.verbatim
-                            len,
-                        }) = self.input.peek()
         {
-                            Some((
+            let event_opener = *event_opener;
-                                if first.len == 2 {
+            let len_opener = *len_opener;
-                                    DisplayMath
+            if usize::from(len_opener) == first.len
-                                } else {
+                && matches!(first.kind, lex::Kind::Seq(Sequence::Backtick))
-                                    InlineMath
-                                },
-                                *len,
-                            ))
-                        } else {
-                            None
-                        }
-                    })
-                    .flatten();
-                if math_opt.is_some() {
-                    self.input.eat(); // backticks
-                }
-                math_opt
-            }
-            lex::Kind::Seq(Sequence::Backtick) => Some((Verbatim, first.len)),
-            _ => None,
-        }?;
-
-        let e_attr = self.events.len();
-        self.push_sp(
-            EventKind::Placeholder,
-            Span::empty_at(self.input.span.start()),
-        );
-        let opener_event = self.events.len();
-        self.push(EventKind::Enter(kind));
-
-        let mut span_inner = self.input.span.empty_after();
-        let mut span_outer = None;
-
-        let mut non_whitespace_first = None;
-        let mut non_whitespace_last = None;
-
-        while let Some(t) = self.input.eat() {
-            if matches!(t.kind, lex::Kind::Seq(Sequence::Backtick)) && t.len == opener_len {
-                if matches!(kind, Verbatim)
-                    && matches!(
-                        self.input.peek().map(|t| &t.kind),
-                        Some(lex::Kind::Open(Delimiter::BraceEqual))
-                    )
             {
-                    let mut ahead = self.input.lexer.chars();
+                let raw_format = self.input.ahead_raw_format();
-                    let mut end = false;
+                let mut span_closer = self.input.span;
-                    let len = (&mut ahead)
+                if let Some(span_format) = raw_format {
-                        .skip(2) // {=
+                    self.events[event_opener].kind = EventKind::Enter(RawFormat);
-                        .take_while(|c| {
+                    self.events[event_opener].span = span_format;
-                            if *c == '{' {
+                    self.input.span = span_format.translate(1);
-                                return false;
+                    span_closer = span_format;
-                            }
+                } else if let Some((non_empty, span_attr)) = self.input.ahead_attributes() {
-                            if *c == '}' {
-                                end = true;
-                            };
-                            !end && !c.is_whitespace()
-                        })
-                        .map(char::len_utf8)
-                        .sum();
-                    if len > 0 && end {
-                        let tok = self.input.eat();
-                        debug_assert_eq!(
-                            tok,
-                            Some(lex::Token {
-                                kind: lex::Kind::Open(Delimiter::BraceEqual),
-                                len: 2,
-                            })
-                        );
-                        self.input.lexer = lex::Lexer::new(ahead);
-                        let span_format = self.input.span.after(len);
-                        kind = RawFormat;
-                        self.events[opener_event].kind = EventKind::Enter(kind);
-                        self.events[opener_event].span = span_format;
-                        self.input.span = span_format.translate(1); // }
-                        span_outer = Some(span_format);
-                    }
-                }
-                break;
-            }
-            if !matches!(t.kind, lex::Kind::Whitespace) {
-                if non_whitespace_first.is_none() {
-                    non_whitespace_first = Some((t.kind, span_inner.end()));
-                }
-                non_whitespace_last = Some((t.kind, span_inner.end() + t.len));
-            }
-            span_inner = span_inner.extend(t.len);
-            self.input.reset_span();
-        }
-
-        if let Some((lex::Kind::Seq(Sequence::Backtick), pos)) = non_whitespace_first {
-            span_inner = span_inner.with_start(pos);
-        }
-        if let Some((lex::Kind::Seq(Sequence::Backtick), pos)) = non_whitespace_last {
-            span_inner = span_inner.with_end(pos);
-        }
-
-        self.push_sp(EventKind::Str, span_inner);
-        self.push_sp(EventKind::Exit(kind), span_outer.unwrap_or(self.input.span));
-
-        if let Some((non_empty, span)) = self.input.ahead_attributes() {
-            self.input.span = span;
                     if non_empty {
+                        let e_attr = event_opener - 1;
                         self.events[e_attr] = Event {
                             kind: EventKind::Attributes,
-                    span,
+                            span: span_attr,
                         };
                     }
+                    self.input.span = span_attr;
+                };
+                let ty_opener = if let EventKind::Enter(ty) = self.events[event_opener].kind {
+                    debug_assert!(matches!(
+                        ty,
+                        Verbatim | RawFormat | InlineMath | DisplayMath
+                    ));
+                    ty
+                } else {
+                    panic!()
+                };
+                if let Some((lex::Kind::Seq(Sequence::Backtick), event_skip)) = non_whitespace_last
+                {
+                    self.events.drain(*event_skip..);
                 }
-
+                self.push_sp(EventKind::Exit(ty_opener), span_closer);
+                self.verbatim = None;
+            } else {
+                // continue verbatim
+                if matches!(first.kind, lex::Kind::Whitespace) {
+                    if !*non_whitespace_encountered
+                        && self.input.peek().map_or(false, |t| {
+                            matches!(
+                                t.kind,
+                                lex::Kind::Seq(Sequence::Backtick) if t.len != len_opener.into(),
+                            )
+                        })
+                    {
+                        return Some(()); // skip whitespace
+                    }
+                } else {
+                    *non_whitespace_encountered = true;
+                    *non_whitespace_last = Some((first.kind, self.events.len() + 1));
+                }
+                self.push(EventKind::Str);
+            };
             Some(())
+        } else {
+            let (ty, len_opener) = match first.kind {
+                lex::Kind::DollarBacktick(l) if first.len - l as usize == 1 => {
+                    Some((InlineMath, l))
+                }
+                lex::Kind::DollarBacktick(l) if first.len - l as usize == 2 => {
+                    Some((DisplayMath, l))
+                }
+                lex::Kind::Seq(Sequence::Backtick) if first.len < 256 => {
+                    Some((Verbatim, first.len as u8))
+                }
+                _ => None,
+            }?;
+            self.push_sp(EventKind::Placeholder, self.input.span.empty_before());
+            self.verbatim = Some(VerbatimState {
+                event_opener: self.events.len(),
+                len_opener,
+                non_whitespace_encountered: false,
+                non_whitespace_last: None,
+            });
+            self.push(EventKind::Enter(ty))
+        }
     }
 
     fn parse_attributes(&mut self, first: &lex::Token) -> Option<()> {
@@ -339,10 +354,11 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
                     self.push(EventKind::Attributes);
                     self.push_sp(EventKind::Enter(Span), span_str.empty_before());
                     self.push_sp(EventKind::Str, span_str);
-                    return self.push_sp(EventKind::Exit(Span), span_str.empty_after());
+                    self.push_sp(EventKind::Exit(Span), span_str.empty_after());
                 } else {
-                    return self.push_sp(EventKind::Placeholder, self.input.span.empty_before());
+                    self.push_sp(EventKind::Placeholder, self.input.span.empty_before());
                 }
+                return Some(());
             }
         }
 
@@ -400,9 +416,9 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
             if end && valid {
                 self.input.lexer = lex::Lexer::new(ahead);
                 self.input.span = self.input.span.after(len);
-                let span = self.input.span;
+                self.push(EventKind::Atom(Symbol));
                 self.input.span = self.input.span.after(1);
-                return self.push_sp(EventKind::Atom(Symbol), span);
+                return Some(());
             }
         }
         None
@@ -452,13 +468,11 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
     }
 
     fn parse_container(&mut self, first: &lex::Token) -> Option<()> {
-        let (delim, dir) = Delim::from_token(first.kind)?;
+        let (delim, dir) = Delim::from_token(first)?;
 
         self.openers
             .iter()
-            .rposition(|(d, _)| {
+            .rposition(|(d, _)| delim.closes(*d))
-                *d == delim || matches!((d, delim), (Delim::Span(..), Delim::Span(..)))
-            })
             .and_then(|o| {
                 if matches!(dir, Dir::Open) {
                     return None;
@@ -479,6 +493,7 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
                 }
 
                 let inner_span = self.events[e_opener].span.between(self.input.span);
+                self.openers.drain(o..);
                 let mut closed = match DelimEventKind::from(d) {
                     DelimEventKind::Container(cont) => {
                         self.events[e_opener].kind = EventKind::Enter(cont);
@@ -490,7 +505,6 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
                     }
                     DelimEventKind::Span(ty) => self.post_span(ty, e_opener),
                 };
-                self.openers.drain(o..);
 
                 if closed.is_some() {
                     let event_closer = &mut self.events.back_mut().unwrap();
@@ -566,7 +580,7 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
 
     fn post_span(&mut self, ty: SpanType, opener_event: usize) -> Option<()> {
         let mut ahead = self.input.lexer.chars();
-        let (kind, span) = match ahead.next() {
+        let kind = match ahead.next() {
             Some(opener @ ('[' | '(')) => {
                 let img = ty == SpanType::Image;
                 let (closer, kind) = match opener {
@@ -587,19 +601,22 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
                     })
                     .map(char::len_utf8)
                     .sum();
-                end.then(|| {
+                if end {
-                    let span = self.input.span.after(len).translate(1);
+                    self.input.span = self.input.span.after(len).translate(1);
-                    (kind, span)
+                    Some(kind)
-                })
+                } else {
+                    None
+                }
             }
             _ => None,
         }?;
 
         self.input.lexer = lex::Lexer::new(ahead);
         self.events[opener_event].kind = EventKind::Enter(kind);
-        self.events[opener_event].span = span;
+        self.events[opener_event].span = self.input.span;
-        self.input.span = span.translate(1);
+        self.push(EventKind::Exit(kind));
-        self.push_sp(EventKind::Exit(kind), span)
+        self.input.span = self.input.span.translate(1);
+        Some(())
     }
 
     fn parse_atom(&mut self, first: &lex::Token) -> Option<()> {
@@ -695,13 +712,13 @@ enum Dir {
 }
 
 impl Delim {
-    fn from_token(kind: lex::Kind) -> Option<(Self, Dir)> {
+    fn from_token(token: &lex::Token) -> Option<(Self, Dir)> {
         use Delim::*;
         use Dir::{Both, Close, Open};
         use Directionality::{Bi, Uni};
         use SpanType::{General, Image};
 
-        match kind {
+        match token.kind {
             lex::Kind::Sym(Symbol::Asterisk) => Some((Strong(Bi), Both)),
             lex::Kind::Sym(Symbol::Underscore) => Some((Emphasis(Bi), Both)),
             lex::Kind::Sym(Symbol::Caret) => Some((Superscript(Bi), Both)),
@@ -732,6 +749,10 @@ impl Delim {
             _ => None,
         }
     }
+
+    fn closes(self, opener: Delim) -> bool {
+        self == opener || matches!((opener, self), (Delim::Span(..), Delim::Span(..)))
+    }
 }
 
 enum DelimEventKind {
@@ -763,6 +784,7 @@ impl<I: Iterator<Item = char> + Clone> Iterator for Parser<I> {
     fn next(&mut self) -> Option<Self::Item> {
         while self.events.is_empty()
             || !self.openers.is_empty()
+            || self.verbatim.is_some()
             || self // for merge or attributes
                 .events
                 .back()
@@ -775,6 +797,20 @@ impl<I: Iterator<Item = char> + Clone> Iterator for Parser<I> {
             }
         }
 
+        // automatically close unclosed verbatim
+        if let Some(VerbatimState { event_opener, .. }) = self.verbatim.take() {
+            let ty_opener = if let EventKind::Enter(ty) = self.events[event_opener].kind {
+                debug_assert!(matches!(
+                    ty,
+                    Verbatim | RawFormat | InlineMath | DisplayMath
+                ));
+                ty
+            } else {
+                panic!()
+            };
+            self.push(EventKind::Exit(ty_opener));
+        }
+
         self.events.pop_front().and_then(|e| {
             match e.kind {
                 EventKind::Str if e.span.is_empty() => self.next(),

src/lex.rs

@@ -21,6 +21,7 @@ pub enum Kind {
     Close(Delimiter),
     Sym(Symbol),
     Seq(Sequence),
+    DollarBacktick(u8),
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -55,7 +56,6 @@ pub enum Symbol {
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Sequence {
     Backtick,
-    Dollar,
     Hyphen,
     Period,
 }
@@ -64,7 +64,6 @@ impl Sequence {
     fn ch(self) -> char {
         match self {
             Self::Backtick => '`',
-            Self::Dollar => '$',
             Self::Period => '.',
             Self::Hyphen => '-',
         }
@@ -207,6 +206,7 @@ impl<I: Iterator<Item = char> + Clone> Lexer<I> {
                     Open(Brace)
                 }
             }
+            '}' => Close(Brace),
             '*' => self.maybe_eat_close_brace(Sym(Asterisk), BraceAsterisk),
             '^' => self.maybe_eat_close_brace(Sym(Caret), BraceCaret),
             '=' => self.maybe_eat_close_brace(Text, BraceEqual),
@@ -236,8 +236,21 @@ impl<I: Iterator<Item = char> + Clone> Lexer<I> {
             ':' => Sym(Colon),
 
             '`' => self.eat_seq(Backtick),
-            '$' => self.eat_seq(Dollar),
             '.' => self.eat_seq(Period),
+            '$' => {
+                self.eat_while(|c| c == '$');
+                let mut n_ticks: u8 = 0;
+                self.eat_while(|c| {
+                    if c == '`' {
+                        if let Some(l) = n_ticks.checked_add(1) {
+                            n_ticks = l;
+                            return true;
+                        }
+                    }
+                    false
+                });
+                DollarBacktick(n_ticks)
+            }
 
             _ => Text,
         };
@@ -383,11 +396,17 @@ mod test {
         test_lex!("`", Seq(Backtick).l(1));
         test_lex!("```", Seq(Backtick).l(3));
         test_lex!(
-            "`$-.",
+            "`-.",
             Seq(Backtick).l(1),
-            Seq(Dollar).l(1),
             Seq(Hyphen).l(1),
             Seq(Period).l(1),
         );
     }
+
+    #[test]
+    fn dollar_backtick() {
+        test_lex!("$`", DollarBacktick(1).l(2));
+        test_lex!("$$$`", DollarBacktick(1).l(4));
+        test_lex!("$$````", DollarBacktick(4).l(6));
+    }
 }

src/span.rs

@@ -30,14 +30,6 @@ impl Span {
         Self::empty_at(self.end())
     }
 
-    pub fn with_start(self, start: usize) -> Self {
-        Self::new(start, self.end())
-    }
-
-    pub fn with_end(self, end: usize) -> Self {
-        Self::new(self.start(), end)
-    }
-
     pub fn with_len(self, len: usize) -> Self {
         Self::by_len(self.start(), len)
     }