Compare commits

...

10 commits

Author SHA1 Message Date
8239b2b51d
Make databake compatible 2024-03-18 18:35:08 -04:00
Noah Hellman
35891f8f49 add release ci action 2024-01-13 00:30:34 +01:00
Noah Hellman
0ea38bf267 Release 0.3.2 2023-09-06 20:18:20 +02:00
Noah Hellman
f43a98478a ci: fix new afl crate name 2023-08-29 18:33:19 +02:00
Noah Hellman
905d2919e3 only allow 1-char alphabetic list markers 2023-08-29 18:33:16 +02:00
Noah Hellman
aff431e227 Release 0.3.1 2023-08-05 14:23:20 +02:00
Noah Hellman
6091f2ea70 html: fix missing <p> tags after ordered lists
resolves #44
2023-08-05 14:05:35 +02:00
Noah Hellman
0586bf6a44 block: rm extra new line after raw blocks
match reference implementation
2023-08-05 14:05:35 +02:00
Noah Hellman
b60650dd0d inline: fix label of empty multi-line ref links
e.g.

    [some
    text][]

    [some text]: url
2023-08-05 14:05:35 +02:00
Noah Hellman
253d1d2d4b prepass: avoid peekable for block iter 2023-08-05 14:05:35 +02:00
18 changed files with 548 additions and 112 deletions

View file

@ -81,7 +81,7 @@ jobs:
run: |
rustup update nightly
rustup default nightly
cargo install afl
cargo install cargo-afl
- name: "Fuzz"
run: |
echo core | sudo tee /proc/sys/kernel/core_pattern

104
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,104 @@
name: release
on:
push:
tags: ["[0-9]+.[0-9]+.[0-9]+*"]
permissions:
contents: write
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
jobs:
create:
name: create release
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.ref_name }}
- name: verify version matches
shell: bash
run: grep -q 'version = "${{ github.ref_name }}"' Cargo.toml || { echo version mismatch >&2 && exit 1; }
- name: create release
run: gh release create ${{ github.ref_name }} --draft --verify-tag --title "Release ${{ github.ref_name }}"
build:
name: build
needs: ['create']
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
target: x86_64-unknown-linux-gnu
- os: ubuntu-latest
target: i686-unknown-linux-musl
- os: macos-latest
target: x86_64-apple-darwin
- os: macos-latest
target: aarch64-apple-darwin
- os: windows-latest
target: x86_64-pc-windows-msvc
- os: windows-latest
target: i686-pc-windows-msvc
steps:
- name: checkout
uses: actions/checkout@v4
with:
ref: ${{ github.ref_name }}
- name: install rust
shell: bash
run: |
rustup update stable
rustup target add ${{ matrix.target }}
- name: build
shell: bash
run: |
if [ -n "${{ matrix.linker }}" ]; then
export RUSTFLAGS="-Clinker=${{ matrix.linker }}"
fi
cargo build --verbose --release --target ${{ matrix.target }}
find .
bin="target/${{ matrix.target }}/release/jotdown"
[ "${{ matrix.os }}" = "windows-latest" ] && bin="$bin.exe"
echo "BIN=$bin" >> $GITHUB_ENV
- name: strip
if: ${{ startsWith(matrix.os, 'ubuntu') }}
run: strip $BIN
- name: set archive name
shell: bash
run: echo "ARCHIVE=jotdown-${{ github.ref_name }}-${{ matrix.target }}" >> $GITHUB_ENV
- name: init archive dir
shell: bash
run: |
mkdir "$ARCHIVE"/
cp "$BIN" "$ARCHIVE"/
cp {COPYING,CHANGELOG.md,README.md} "$ARCHIVE"/
- name: archive (win)
if: ${{ startsWith(matrix.os, 'windows') }}
shell: bash
run: |
7z a "$ARCHIVE.zip" "$ARCHIVE"
echo "ASSET=$ARCHIVE.zip" >> $GITHUB_ENV
- name: archive (unix)
if: ${{ ! startsWith(matrix.os, 'windows') }}
shell: bash
run: |
tar czf "$ARCHIVE.tar.gz" "$ARCHIVE"
echo "ASSET=$ARCHIVE.tar.gz" >> $GITHUB_ENV
- name: Upload release archive
shell: bash
run: |
gh release upload ${{ github.ref_name }} ${{ env.ASSET }}

View file

@ -1,3 +1,24 @@
## [0.3.2](https://github.com/hellux/jotdown/releases/tag/0.3.2) - 2023-09-06
### Changed
- Alphabetic list markers can only be one character long.
## [0.3.1](https://github.com/hellux/jotdown/releases/tag/0.3.1) - 2023-08-05
### Changed
- Block parser performance improved, up to 15% faster.
- Last `unsafe` block removed (#5).
### Fixed
- Do not require indent for continuing footnotes.
- Transfer classes from reference definitions to links.
- Allow line breaks in reference links (still match reference label).
- Remove excess newline after raw blocks.
- HTML renderer: fix missing `<p>` tags after ordered lists (#44).
## [0.3.0](https://github.com/hellux/jotdown/releases/tag/0.3.0) - 2023-05-16
### Added

62
Cargo.lock generated
View file

@ -162,6 +162,29 @@ dependencies = [
"itertools",
]
[[package]]
name = "databake"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82175d72e69414ceafbe2b49686794d3a8bed846e0d50267355f83ea8fdd953a"
dependencies = [
"databake-derive",
"proc-macro2",
"quote",
]
[[package]]
name = "databake-derive"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "377af281d8f23663862a7c84623bc5dcf7f8c44b13c7496a590bdc157f941a43"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.16",
"synstructure",
]
[[package]]
name = "either"
version = "1.8.1"
@ -263,11 +286,14 @@ dependencies = [
[[package]]
name = "jotdown"
version = "0.3.0"
version = "0.3.2"
dependencies = [
"databake",
]
[[package]]
name = "jotdown_wasm"
version = "0.3.0"
version = "0.3.2"
dependencies = [
"git2",
"jotdown",
@ -370,9 +396,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]]
name = "proc-macro2"
version = "1.0.56"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
dependencies = [
"unicode-ident",
]
@ -433,7 +459,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 1.0.107",
]
[[package]]
@ -458,6 +484,28 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "synstructure"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.16",
]
[[package]]
name = "test-html-ref"
version = "0.1.0"
@ -573,7 +621,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn",
"syn 1.0.107",
"wasm-bindgen-shared",
]
@ -595,7 +643,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 1.0.107",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]

View file

@ -2,7 +2,7 @@
name = "jotdown"
description = "A parser for the Djot markup language"
authors = ["Noah Hellman <noah@hllmn.net>"]
version = "0.3.0"
version = "0.3.2"
license = "MIT"
edition = "2021"
keywords = ["djot", "markup"]
@ -35,10 +35,14 @@ exclude = [
[[bin]]
name = "jotdown"
required-features = ["html"]
required-features = ["html", "parser"]
doc = false
[features]
default = ["html"]
html = [] # html renderer and minimal cli binary
deterministic = [] # for stable fuzzing
parser = []
[dependencies]
databake = { version = "0.1.7", features = ["derive"] }

View file

@ -3,7 +3,7 @@ name = "jotdown_wasm"
description = "Web demo of Jotdown"
authors = ["Noah Hellman <noah@hllmn.net>"]
license = "MIT"
version = "0.3.0"
version = "0.3.2"
edition = "2021"
homepage = "https://hllmn.net/projects/jotdown"
repository = "https://github.com/hellux/jotdown"

View file

@ -1,13 +1,17 @@
use databake::Bake;
use crate::CowStr;
use std::fmt;
use std::{borrow::Cow, fmt};
/// Parse attributes, assumed to be valid.
#[cfg(feature = "parser")]
pub(crate) fn parse(src: &str) -> Attributes {
let mut a = Attributes::new();
a.parse(src);
a
}
#[cfg(feature = "parser")]
pub fn valid(src: &str) -> usize {
use State::*;
@ -31,7 +35,8 @@ pub fn valid(src: &str) -> usize {
/// Stores an attribute value that supports backslash escapes of ASCII punctuation upon displaying,
/// without allocating.
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq, Bake)]
#[databake(path = jotdown)]
pub struct AttributeValue<'s> {
raw: CowStr<'s>,
}
@ -118,8 +123,9 @@ impl<'s> Iterator for AttributeValueParts<'s> {
// Attributes are relatively rare, we choose to pay 8 bytes always and sometimes an extra
// indirection instead of always 24 bytes.
#[allow(clippy::box_vec)]
#[derive(Clone, PartialEq, Eq, Default)]
pub struct Attributes<'s>(Option<Box<Vec<(&'s str, AttributeValue<'s>)>>>);
#[derive(Clone, PartialEq, Eq, Default, Bake)]
#[databake(path = jotdown)]
pub struct Attributes<'s>(pub Option<Cow<'s, [(&'s str, AttributeValue<'s>)]>>);
impl<'s> Attributes<'s> {
/// Create an empty collection.
@ -129,11 +135,13 @@ impl<'s> Attributes<'s> {
}
#[must_use]
#[cfg(feature = "parser")]
pub(crate) fn take(&mut self) -> Self {
Self(self.0.take())
}
/// Parse and append attributes, assumed to be valid.
#[cfg(feature = "parser")]
pub(crate) fn parse(&mut self, input: &'s str) {
let mut parser = Parser::new(self.take());
parser.parse(input);
@ -141,12 +149,13 @@ impl<'s> Attributes<'s> {
}
/// Combine all attributes from both objects, prioritizing self on conflicts.
#[cfg(feature = "parser")]
pub(crate) fn union(&mut self, other: Self) {
if let Some(attrs0) = &mut self.0 {
if let Some(mut attrs1) = other.0 {
for (key, val) in attrs1.drain(..) {
for (key, val) in attrs1.to_mut().drain(..) {
if key == "class" || !attrs0.iter().any(|(k, _)| *k == key) {
attrs0.push((key, val));
attrs0.to_mut().push((key, val));
}
}
}
@ -170,7 +179,7 @@ impl<'s> Attributes<'s> {
let attrs = self.0.as_mut().unwrap();
if let Some(i) = attrs.iter().position(|(k, _)| *k == key) {
let prev = &mut attrs[i].1;
let prev = &mut attrs.to_mut()[i].1;
if key == "class" {
match val.raw {
CowStr::Borrowed(s) => prev.extend(s),
@ -184,7 +193,7 @@ impl<'s> Attributes<'s> {
i
} else {
let i = attrs.len();
attrs.push((key, val));
attrs.to_mut().push((key, val));
i
}
}
@ -238,10 +247,12 @@ impl<'s> std::fmt::Debug for Attributes<'s> {
}
#[derive(Clone)]
#[cfg(feature = "parser")]
pub struct Validator {
state: State,
}
#[cfg(feature = "parser")]
impl Validator {
pub fn new() -> Self {
Self {
@ -274,12 +285,14 @@ impl Validator {
///
/// Input is assumed to contain a valid series of attribute sets, the attributes are added as they
/// are encountered.
#[cfg(feature = "parser")]
pub struct Parser<'s> {
attrs: Attributes<'s>,
i_prev: usize,
state: State,
}
#[cfg(feature = "parser")]
impl<'s> Parser<'s> {
pub fn new(attrs: Attributes<'s>) -> Self {
Self {
@ -310,7 +323,7 @@ impl<'s> Parser<'s> {
Identifier => self.attrs.insert("id", content.into()),
Key => self.i_prev = self.attrs.insert_pos(content, "".into()),
Value | ValueQuoted | ValueContinued => {
self.attrs.0.as_mut().unwrap()[self.i_prev]
self.attrs.0.as_mut().unwrap().to_mut()[self.i_prev]
.1
.extend(&content[usize::from(matches!(st, ValueQuoted))..]);
}
@ -338,6 +351,7 @@ impl<'s> Parser<'s> {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg(feature = "parser")]
enum State {
Start,
Whitespace,
@ -357,6 +371,7 @@ enum State {
Invalid,
}
#[cfg(feature = "parser")]
impl State {
fn step(self, c: u8) -> State {
use State::*;
@ -399,11 +414,12 @@ impl State {
}
}
#[cfg(feature = "parser")]
pub fn is_name(c: u8) -> bool {
c.is_ascii_alphanumeric() || matches!(c, b':' | b'_' | b'-')
}
#[cfg(test)]
#[cfg(all(test, feature = "parser"))]
mod test {
macro_rules! test_attr {
($src:expr $(,$($av:expr),* $(,)?)?) => {

View file

@ -353,7 +353,7 @@ impl<'s> TreeParser<'s> {
span_end: Range<usize>,
mut lines: &mut [Range<usize>],
) {
if let Kind::Fenced { indent, .. } = k {
if let Kind::Fenced { indent, spec, .. } = k {
for line in lines.iter_mut() {
let indent_line = self.src.as_bytes()[line.clone()]
.iter()
@ -361,6 +361,14 @@ impl<'s> TreeParser<'s> {
.count();
line.start += (*indent).min(indent_line);
}
// trim ending whitespace of raw block
if spec.starts_with('=') {
let l = lines.len();
if l > 0 {
lines[l - 1] = self.trim_end(lines[l - 1].clone());
}
}
} else {
// trim starting whitespace of each inline
for line in lines.iter_mut() {
@ -1011,21 +1019,22 @@ impl<'s> IdentifiedBlock<'s> {
let numbering = if first.is_ascii_digit() {
Decimal
} else if first.is_ascii_lowercase() {
AlphaLower
} else if first.is_ascii_uppercase() {
AlphaUpper
} else if is_roman_lower_digit(first) {
RomanLower
} else if is_roman_upper_digit(first) {
RomanUpper
} else if first.is_ascii_lowercase() {
AlphaLower
} else if first.is_ascii_uppercase() {
AlphaUpper
} else {
return None;
};
let max_len = match numbering {
AlphaLower | AlphaUpper => 1,
Decimal => 19,
AlphaLower | AlphaUpper | RomanLower | RomanUpper => 13,
RomanLower | RomanUpper => 13,
};
let chars_num = chars.clone();
@ -1057,17 +1066,6 @@ impl<'s> IdentifiedBlock<'s> {
};
let len_style = usize::from(start_paren) + 1;
let chars_num = std::iter::once(first).chain(chars_num.take(len_num - 1));
let numbering = if matches!(numbering, AlphaLower)
&& chars_num.clone().all(is_roman_lower_digit)
{
RomanLower
} else if matches!(numbering, AlphaUpper) && chars_num.clone().all(is_roman_upper_digit) {
RomanUpper
} else {
numbering
};
if chars.next().map_or(true, |c| c.is_ascii_whitespace()) {
Some((numbering, style, len_num + len_style))
} else {
@ -3121,16 +3119,6 @@ mod test {
"I.",
1
);
test_block!(
"IJ. abc\n",
Kind::ListItem {
indent: 0,
ty: Ordered(AlphaUpper, Period),
last_blankline: false,
},
"IJ.",
1
);
test_block!(
"(a) abc\n",
Kind::ListItem {

View file

@ -295,17 +295,13 @@ impl<'s> Writer<'s> {
}
match c {
Container::Blockquote => out.write_str("</blockquote>")?,
Container::List {
kind: ListKind::Unordered | ListKind::Task,
..
} => {
Container::List { kind, .. } => {
self.list_tightness.pop();
out.write_str("</ul>")?;
match kind {
ListKind::Unordered | ListKind::Task => out.write_str("</ul>")?,
ListKind::Ordered { .. } => out.write_str("</ol>")?,
}
}
Container::List {
kind: ListKind::Ordered { .. },
..
} => out.write_str("</ol>")?,
Container::ListItem | Container::TaskListItem { .. } => {
out.write_str("</li>")?;
}

View file

@ -732,41 +732,36 @@ impl<'s> Parser<'s> {
image,
} => {
let span_spec = self.events[e_opener].span.end..self.input.span.start;
let multiline =
let multiline_spec =
self.events[e_opener].span.start < self.input.span_line.start;
let spec: CowStr = if span_spec.is_empty() && !inline {
let span_spec = self.events[event_span].span.end
..self.events[e_opener - 1].span.start;
let events_text = self
.events
.iter()
.skip(event_span + 1)
.take(e_opener - event_span - 2);
if multiline
|| events_text.clone().any(|ev| {
!matches!(ev.kind, EventKind::Str | EventKind::Atom(..))
})
{
let mut spec = String::new();
let mut span = 0..0;
for ev in events_text.filter(|ev| {
matches!(ev.kind, EventKind::Str | EventKind::Atom(..))
}) {
if span.end == ev.span.start {
span.end = ev.span.end;
} else {
spec.push_str(&self.input.src[span.clone()]);
span = ev.span.clone();
}
let mut spec = String::new();
let mut span = 0..0;
for ev in events_text.filter(|ev| {
matches!(ev.kind, EventKind::Str | EventKind::Atom(..))
&& !matches!(ev.kind, EventKind::Atom(Escape))
}) {
if matches!(ev.kind, EventKind::Atom(Softbreak | Hardbreak)) {
spec.push_str(&self.input.src[span.clone()]);
spec.push(' ');
span = ev.span.end..ev.span.end;
} else if span.end == ev.span.start {
span.end = ev.span.end;
} else {
spec.push_str(&self.input.src[span.clone()]);
span = ev.span.clone();
}
spec.push_str(&self.input.src[span]);
spec.into()
} else {
self.input.src[span_spec].into()
}
} else if multiline {
spec.push_str(&self.input.src[span]);
spec.into()
} else if multiline_spec {
let mut spec = String::new();
let mut first_part = true;
let mut span =

View file

@ -49,19 +49,25 @@
#![allow(clippy::blocks_in_if_conditions)]
use std::fmt;
#[cfg(feature = "parser")]
use std::fmt::Write as FmtWrite;
use std::io;
#[cfg(feature = "parser")]
use std::ops::Range;
#[cfg(feature = "html")]
pub mod html;
mod attr;
#[cfg(feature = "parser")]
mod block;
#[cfg(feature = "parser")]
mod inline;
#[cfg(feature = "parser")]
mod lex;
pub use attr::{AttributeValue, AttributeValueParts, Attributes};
use databake::Bake;
type CowStr<'s> = std::borrow::Cow<'s, str>;
@ -197,7 +203,8 @@ impl<'s> AsRef<Event<'s>> for &Event<'s> {
/// multiple events. [`Container`] elements are represented by a [`Event::Start`] followed by
/// events representing its content, and finally a [`Event::End`]. Atomic elements without any
/// inside elements are represented by a single event.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Bake)]
#[databake(path = jotdown)]
pub enum Event<'s> {
/// Start of a container.
Start(Container<'s>, Attributes<'s>),
@ -244,7 +251,8 @@ pub enum Event<'s> {
/// - inline, may only contain inline elements,
/// - block leaf, may only contain inline elements,
/// - block container, may contain any block-level elements.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Bake)]
#[databake(path = jotdown)]
pub enum Container<'s> {
/// A blockquote element.
Blockquote,
@ -398,7 +406,8 @@ impl<'s> Container<'s> {
}
/// Alignment of a table column.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Bake)]
#[databake(path = jotdown)]
pub enum Alignment {
Unspecified,
Left,
@ -407,7 +416,8 @@ pub enum Alignment {
}
/// The type of an inline span link.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Bake)]
#[databake(path = jotdown)]
pub enum SpanLinkType {
/// E.g. `[text](url)`
Inline,
@ -418,7 +428,8 @@ pub enum SpanLinkType {
}
/// The type of an inline link.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Bake)]
#[databake(path = jotdown)]
pub enum LinkType {
/// E.g. `[text](url)`.
Span(SpanLinkType),
@ -429,7 +440,8 @@ pub enum LinkType {
}
/// The type of a list.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Bake)]
#[databake(path = jotdown)]
pub enum ListKind {
/// A bullet list.
Unordered,
@ -444,7 +456,8 @@ pub enum ListKind {
}
/// Numbering type of an ordered list.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Bake)]
#[databake(path = jotdown)]
pub enum OrderedListNumbering {
/// Decimal numbering, e.g. `1)`.
Decimal,
@ -459,7 +472,8 @@ pub enum OrderedListNumbering {
}
/// Style of an ordered list.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Bake)]
#[databake(path = jotdown)]
pub enum OrderedListStyle {
/// Number is followed by a period, e.g. `1.`.
Period,
@ -470,6 +484,7 @@ pub enum OrderedListStyle {
}
impl OrderedListNumbering {
#[cfg(feature = "parser")]
fn parse_number(self, n: &str) -> u64 {
match self {
Self::Decimal => n.parse().unwrap(),
@ -524,6 +539,7 @@ impl OrderedListNumbering {
}
impl OrderedListStyle {
#[cfg(feature = "parser")]
fn number(self, marker: &str) -> &str {
&marker[usize::from(matches!(self, Self::ParenParen))..marker.len() - 1]
}
@ -534,9 +550,9 @@ type Map<K, V> = std::collections::HashMap<K, V>;
#[cfg(feature = "deterministic")]
type Map<K, V> = std::collections::BTreeMap<K, V>;
#[cfg(not(feature = "deterministic"))]
#[cfg(all(not(feature = "deterministic"), feature = "parser"))]
type Set<T> = std::collections::HashSet<T>;
#[cfg(feature = "deterministic")]
#[cfg(all(feature = "deterministic", feature = "parser"))]
type Set<T> = std::collections::BTreeSet<T>;
/// A parser that generates [`Event`]s from a Djot document.
@ -548,6 +564,7 @@ type Set<T> = std::collections::BTreeSet<T>;
///
/// It is possible to clone the parser to e.g. avoid performing the block parsing multiple times.
#[derive(Clone)]
#[cfg(feature = "parser")]
pub struct Parser<'s> {
src: &'s str,
@ -572,6 +589,7 @@ pub struct Parser<'s> {
}
#[derive(Clone)]
#[cfg(feature = "parser")]
struct Heading {
/// Location of heading in src.
location: u32,
@ -585,6 +603,7 @@ struct Heading {
/// Because of potential future references, an initial pass is required to obtain all definitions.
#[derive(Clone)]
#[cfg(feature = "parser")]
struct PrePass<'s> {
/// Link definitions and their attributes.
link_definitions: Map<&'s str, (CowStr<'s>, attr::Attributes<'s>)>,
@ -594,53 +613,61 @@ struct PrePass<'s> {
headings_lex: Vec<usize>,
}
#[cfg(feature = "parser")]
impl<'s> PrePass<'s> {
#[must_use]
fn new(
src: &'s str,
blocks: std::slice::Iter<block::Event<'s>>,
mut blocks: std::slice::Iter<block::Event<'s>>,
inline_parser: &mut inline::Parser<'s>,
) -> Self {
let mut link_definitions = Map::new();
let mut headings: Vec<Heading> = Vec::new();
let mut used_ids: Set<String> = Set::new();
let mut blocks = blocks.peekable();
let mut attr_prev: Option<Range<usize>> = None;
while let Some(e) = blocks.next() {
match e.kind {
block::EventKind::Enter(block::Node::Leaf(block::Leaf::LinkDefinition {
label,
})) => {
fn next_is_inline(
bs: &mut std::iter::Peekable<std::slice::Iter<block::Event>>,
) -> bool {
matches!(bs.peek().map(|e| &e.kind), Some(block::EventKind::Inline))
}
// All link definition tags have to be obtained initially, as references can
// appear before the definition.
let attrs = attr_prev
.as_ref()
.map_or_else(Attributes::new, |sp| attr::parse(&src[sp.clone()]));
let url = if !next_is_inline(&mut blocks) {
"".into()
} else {
let start = src[blocks.next().as_ref().unwrap().span.clone()]
.trim_matches(|c: char| c.is_ascii_whitespace());
if !next_is_inline(&mut blocks) {
start.into()
} else {
let url = if let Some(block::Event {
kind: block::EventKind::Inline,
span,
}) = blocks.next()
{
let start =
src[span.clone()].trim_matches(|c: char| c.is_ascii_whitespace());
if let Some(block::Event {
kind: block::EventKind::Inline,
span,
}) = blocks.next()
{
let mut url = start.to_string();
while next_is_inline(&mut blocks) {
url.push_str(
src[span.clone()].trim_matches(|c: char| c.is_ascii_whitespace()),
);
while let Some(block::Event {
kind: block::EventKind::Inline,
span,
}) = blocks.next()
{
url.push_str(
src[blocks.next().as_ref().unwrap().span.clone()]
src[span.clone()]
.trim_matches(|c: char| c.is_ascii_whitespace()),
);
}
url.into()
url.into() // owned
} else {
start.into() // borrowed
}
} else {
"".into() // static
};
link_definitions.insert(label, (url, attrs));
}
@ -775,6 +802,7 @@ impl<'s> PrePass<'s> {
}
}
#[cfg(feature = "parser")]
impl<'s> Parser<'s> {
#[must_use]
pub fn new(src: &'s str) -> Self {
@ -1158,6 +1186,7 @@ impl<'s> Parser<'s> {
}
}
#[cfg(feature = "parser")]
impl<'s> Iterator for Parser<'s> {
type Item = Event<'s>;
@ -1170,10 +1199,12 @@ impl<'s> Iterator for Parser<'s> {
/// event within the input.
///
/// See the documentation of [`Parser::into_offset_iter`] for more information.
#[cfg(feature = "parser")]
pub struct OffsetIter<'s> {
parser: Parser<'s>,
}
#[cfg(feature = "parser")]
impl<'s> Iterator for OffsetIter<'s> {
type Item = (Event<'s>, Range<usize>);
@ -1183,6 +1214,7 @@ impl<'s> Iterator for OffsetIter<'s> {
}
#[cfg(test)]
#[cfg(feature = "parser")]
mod test {
use super::Attributes;
use super::Container::*;
@ -1527,7 +1559,39 @@ mod test {
test_parse!(
"``` =html\n<table>\n```",
Start(RawBlock { format: "html" }, Attributes::new()),
Str("<table>\n".into()),
Str("<table>".into()),
End(RawBlock { format: "html" }),
);
}
#[test]
fn raw_block_whitespace() {
test_parse!(
concat!(
"```=html\n", //
"<tag1>\n", //
"<tag2>\n", //
"```\n", //
"\n", //
"paragraph\n", //
"\n", //
"```=html\n", //
"</tag2>\n", //
"</tag1>\n", //
"```\n", //
),
Start(RawBlock { format: "html" }, Attributes::new()),
Str("<tag1>\n".into()),
Str("<tag2>".into()),
End(RawBlock { format: "html" }),
Blankline,
Start(Paragraph, Attributes::new()),
Str("paragraph".into()),
End(Paragraph),
Blankline,
Start(RawBlock { format: "html" }, Attributes::new()),
Str("</tag2>\n".into()),
Str("</tag1>".into()),
End(RawBlock { format: "html" }),
);
}
@ -1690,6 +1754,46 @@ mod test {
);
}
#[test]
fn link_reference_multiline_empty() {
test_parse!(
concat!(
"> [a\n", //
"> b][]\n", //
"> [a\\\n", //
"> b][]\n", //
"\n", //
"[a b]: url\n", //
),
Start(Blockquote, Attributes::new()),
Start(Paragraph, Attributes::new()),
Start(
Link("url".into(), LinkType::Span(SpanLinkType::Reference)),
Attributes::new()
),
Str("a".into()),
Softbreak,
Str("b".into()),
End(Link("url".into(), LinkType::Span(SpanLinkType::Reference))),
Softbreak,
Start(
Link("url".into(), LinkType::Span(SpanLinkType::Reference)),
Attributes::new()
),
Str("a".into()),
Escape,
Hardbreak,
Str("b".into()),
End(Link("url".into(), LinkType::Span(SpanLinkType::Reference))),
End(Paragraph),
End(Blockquote),
Blankline,
Start(LinkDefinition { label: "a b" }, Attributes::new()),
Str("url".into()),
End(LinkDefinition { label: "a b" }),
);
}
#[test]
fn link_definition_multiline() {
test_parse!(

1
tests/html-ref/ref.rs Normal file
View file

@ -0,0 +1 @@

View file

@ -0,0 +1,66 @@
use crate::compare;
// Footnote references may appear within a footnote.
#[test]
fn test_1c8325a() {
let src = r##"[^a]
[^a]: a[^b][^c]
[^b]: b
"##;
let expected = r##"<p><a id="fnref1" href="#fn1" role="doc-noteref"><sup>1</sup></a></p>
<section role="doc-endnotes">
<hr>
<ol>
<li id="fn1">
<p>a<a id="fnref2" href="#fn2" role="doc-noteref"><sup>2</sup></a><a id="fnref3" href="#fn3" role="doc-noteref"><sup>3</sup></a><a href="#fnref1" role="doc-backlink"></a></p>
</li>
<li id="fn2">
<p>b<a href="#fnref2" role="doc-backlink"></a></p>
</li>
<li id="fn3">
<p><a href="#fnref3" role="doc-backlink"></a></p>
</li>
</ol>
</section>
"##;
compare!(src, expected);
}
// Footnote references in unreferenced footnotes are ignored.
#[test]
fn test_9eab5c8() {
let src = r##"para
[^a]: a[^b][^c]
[^b]: b
"##;
let expected = r##"<p>para</p>
"##;
compare!(src, expected);
}
// Footnotes may appear within footnotes.
#[test]
fn test_041f54c() {
let src = r##"[^b]
[^a]
[^a]: [^b]: inner
"##;
let expected = r##"<p><a id="fnref1" href="#fn1" role="doc-noteref"><sup>1</sup></a>
<a id="fnref2" href="#fn2" role="doc-noteref"><sup>2</sup></a></p>
<section role="doc-endnotes">
<hr>
<ol>
<li id="fn1">
<p>inner<a href="#fnref1" role="doc-backlink"></a></p>
</li>
<li id="fn2">
<p><a href="#fnref2" role="doc-backlink"></a></p>
</li>
</ol>
</section>
"##;
compare!(src, expected);
}

27
tests/html-ut/ut/lists.rs Normal file
View file

@ -0,0 +1,27 @@
use crate::compare;
#[test]
fn test_fefa2dc() {
let src = r##"1. item
para
"##;
let expected = r##"<ol>
<li>
item
</li>
</ol>
<p>para</p>
"##;
compare!(src, expected);
}
// Only single letter alphabetic list markers.
#[test]
fn test_2a0aa95() {
let src = r##"word. Continuing paragraph.
"##;
let expected = r##"<p>word. Continuing paragraph.</p>
"##;
compare!(src, expected);
}

View file

@ -0,0 +1,20 @@
```
1. item
para
.
<ol>
<li>
item
</li>
</ol>
<p>para</p>
```
Only single letter alphabetic list markers.
```
word. Continuing paragraph.
.
<p>word. Continuing paragraph.</p>
```

3
tests/html-ut/ut/mod.rs Normal file
View file

@ -0,0 +1,3 @@
mod footnotes;
mod lists;
mod raw_blocks;

View file

@ -0,0 +1,24 @@
use crate::compare;
#[test]
fn test_bf9dbab() {
let src = r##"```=html
<tag1>
<tag2>
```
paragraph
```=html
</tag2>
</tag1>
```
"##;
let expected = r##"<tag1>
<tag2>
<p>paragraph</p>
</tag2>
</tag1>
"##;
compare!(src, expected);
}

View file

@ -0,0 +1,19 @@
````
```=html
<tag1>
<tag2>
```
paragraph
```=html
</tag2>
</tag1>
```
.
<tag1>
<tag2>
<p>paragraph</p>
</tag2>
</tag1>
````