diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e9eb01f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,80 @@ +name: ci + +on: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + name: Build and run tests + runs-on: ubuntu-latest + strategy: + matrix: + toolchain: + - "1.56" + - stable + steps: + - name: "Checkout repo" + uses: actions/checkout@v3 + - name: "Setup toolchain" + run: | + rustup update ${{ matrix.toolchain }} + rustup default ${{ matrix.toolchain }} + - name: "Build" + run: | + make all + cargo build --workspace --no-default-features + - name: "Run tests" + env: + RUSTDOCFLAGS: -D warnings + run: | + make check + suite: + name: Build and run external tests + runs-on: ubuntu-latest + steps: + - name: "Checkout repo" + uses: actions/checkout@v3 + - name: "Setup toolchain" + run: | + rustup update 1.56 + rustup default 1.56 + - name: "Run unit tests" + run: make suite + - name: "Setup node" + uses: actions/setup-node@v3 + with: + node-version: 18 + - name: "Compare benchmark files" + run: make suite_bench + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: "Checkout" + uses: actions/checkout@v3 + - name: "Setup toolchain" + run: | + rustup update 1.56 + rustup default 1.56 + rustup component add rustfmt + rustup component add clippy + - name: "Check linting" + run: make lint + fuzz: + name: Fuzz + runs-on: ubuntu-latest + steps: + - name: "Checkout" + uses: actions/checkout@v3 + - name: "Setup toolchain" + run: | + rustup update nightly + rustup default nightly + cargo install afl + - name: "Fuzz" + run: | + echo core | sudo tee /proc/sys/kernel/core_pattern + make afl_quick diff --git a/.gitmodules b/.gitmodules index cb3429a..72c8292 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "jgm/djot.js"] path = modules/djot.js - url = git@github.com:jgm/djot.js.git + url = https://github.com/jgm/djot.js.git diff --git a/Cargo.lock b/Cargo.lock index bae5c3b..913ff3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,130 +2,22 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "afl" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c0f4180b6d7095a1c3130f1aadead4ece7fb3094dc724c701adb30a92a95228" -dependencies = [ - "cc", - "clap", - "libc", - "rustc_version", - "xdg", -] - -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bumpalo" version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" - [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags", - "strsim", - "textwrap", - "unicode-width", - "vec_map", -] - -[[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "getrandom" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "jotdown" version = "0.1.0" -[[package]] -name = "jotdown-afl" -version = "0.1.0" -dependencies = [ - "afl", - "jotdown", -] - [[package]] name = "jotdown_wasm" version = "0.1.0" @@ -144,12 +36,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "libc" -version = "0.2.139" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" - [[package]] name = "log" version = "0.4.17" @@ -183,47 +69,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall", - "thiserror", -] - -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - -[[package]] -name = "semver" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" - -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "syn" version = "1.0.107" @@ -235,59 +80,12 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "thiserror" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "unicode-ident" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - [[package]] name = "wasm-bindgen" version = "0.2.84" @@ -351,34 +149,3 @@ dependencies = [ "js-sys", "wasm-bindgen", ] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "xdg" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4583db5cbd4c4c0303df2d15af80f0539db703fa1c68802d4cbbd2dd0f88f6" -dependencies = [ - "dirs", -] diff --git a/Cargo.toml b/Cargo.toml index c59bb97..66654eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,8 @@ exclude = [ [workspace] members = [ "examples/jotdown_wasm", +] +exclude = [ "tests/afl", ] diff --git a/Makefile b/Makefile index 1a9a93c..0054e16 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,30 @@ .POSIX: +all: jotdown docs + cargo build --workspace + +jotdown: target/release/jotdown + cp $< $@ + +target/release/jotdown: + cargo build --release + +.PHONY: +docs: + cargo doc --no-deps --workspace + +.PHONY: lint +lint: + cargo fmt --all -- --check + cargo clippy -- -D warnings + cargo clippy --no-default-features -- -D warnings + cargo clippy --all-features -- -D warnings + +.PHONY: check +check: + cargo test --workspace + cargo test --workspace --no-default-features + .PHONY: suite suite: git submodule update --init modules/djot.js @@ -36,13 +61,20 @@ afl: rm -rf tests/afl/out (cd tests/afl && \ cargo afl build --release --config profile.release.debug-assertions=true && \ - (AFL_NO_UI=1 cargo afl fuzz -i in -o out -Mm ../../target/release/${AFL_TARGET} &) && \ + (AFL_NO_UI=1 cargo afl fuzz -i in -o out -Mm target/release/${AFL_TARGET} &) && \ for i in $$(seq $$((${AFL_JOBS} - 1))); do \ - AFL_NO_UI=1 cargo afl fuzz -i in -o out -Ss$$i ../../target/release/${AFL_TARGET} & \ + AFL_NO_UI=1 cargo afl fuzz -i in -o out -Ss$$i target/release/${AFL_TARGET} & \ done; \ trap - EXIT;\ cat) # keep process alive for trap +afl_quick: + rm -rf tests/afl/out + (cd tests/afl && \ + cargo afl build --release --config profile.release.debug-assertions=true && \ + AFL_NO_UI=1 AFL_BENCH_UNTIL_CRASH=1 \ + cargo afl fuzz -i in -o out -V 60 target/release/${AFL_TARGET}) + afl_crash: set +e; \ for f in $$(find tests/afl/out -path '*/${AFL_TARGET_CRASH}/id*'); do \ diff --git a/src/attr.rs b/src/attr.rs index 4f5a7b1..e724812 100644 --- a/src/attr.rs +++ b/src/attr.rs @@ -27,6 +27,7 @@ pub fn valid>(chars: I) -> (usize, bool) { /// A collection of attributes, i.e. a key-value map. // Attributes are relatively rare, we choose to pay 8 bytes always and sometimes an extra // indirection instead of always 24 bytes. +#[allow(clippy::box_vec)] #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct Attributes<'s>(Option)>>>); diff --git a/src/block.rs b/src/block.rs index 8744521..c7f5692 100644 --- a/src/block.rs +++ b/src/block.rs @@ -351,13 +351,13 @@ impl<'s> TreeParser<'s> { }); if let ListItem(ty) = c { - if self + let same_depth = self .open_lists .last() .map_or(true, |OpenList { depth, .. }| { usize::from(*depth) < self.tree.depth() - }) - { + }); + if same_depth { let tight = true; let node = self.tree.enter( Node::Container(Container::List(ListKind { ty, tight })), diff --git a/src/inline.rs b/src/inline.rs index 3e1cb77..dd9a818 100644 --- a/src/inline.rs +++ b/src/inline.rs @@ -442,11 +442,10 @@ impl + Clone> Parser { if matches!(dir, Dir::Open) { return None; } - if matches!(dir, Dir::Both) - && self.events.back().map_or(false, |ev| { - matches!(ev.kind, EventKind::Whitespace | EventKind::Atom(Softbreak)) - }) - { + let whitespace_after = self.events.back().map_or(false, |ev| { + matches!(ev.kind, EventKind::Whitespace | EventKind::Atom(Softbreak)) + }); + if matches!(dir, Dir::Both) && whitespace_after { return None; } @@ -791,6 +790,7 @@ impl + Clone> Iterator for Parser { type Item = Event; fn next(&mut self) -> Option { + #[allow(clippy::blocks_in_if_conditions)] while self.events.is_empty() || !self.openers.is_empty() || self // for merge or attributes @@ -813,12 +813,13 @@ impl + Clone> Iterator for Parser { EventKind::Str | EventKind::Whitespace => { // merge str events let mut span = e.span; - while self.events.front().map_or(false, |e| { + let should_merge = |e: &Event, span: Span| { matches!( e.kind, EventKind::Str | EventKind::Whitespace | EventKind::Placeholder ) && span.end() == e.span.start() - }) { + }; + while self.events.front().map_or(false, |e| should_merge(e, span)) { let ev = self.events.pop_front().unwrap(); span = span.union(ev.span); } diff --git a/src/lib.rs b/src/lib.rs index ca1aa4e..bd81e3f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -241,7 +241,8 @@ pub enum Container<'s> { impl<'s> Container<'s> { /// Is a block element. - fn is_block(&self) -> bool { + #[must_use] + pub fn is_block(&self) -> bool { match self { Self::Blockquote | Self::List { .. } @@ -278,7 +279,8 @@ impl<'s> Container<'s> { } /// Is a block element that may contain children blocks. - fn is_block_container(&self) -> bool { + #[must_use] + pub fn is_block_container(&self) -> bool { match self { Self::Blockquote | Self::List { .. } diff --git a/tests/bench/Makefile b/tests/bench/Makefile index f99adad..ebd4893 100644 --- a/tests/bench/Makefile +++ b/tests/bench/Makefile @@ -10,17 +10,19 @@ mod.rs: ${TEST_DJ} html echo "use crate::suite_test;" > $@ for name in ${TEST}; do \ name_snake=$$(basename -a $$name | sed 's/-/_/g'); \ - echo "#[test]" >> $@; \ - echo "fn test_$$name_snake() {" >> $@; \ - printf ' let src = r###"' >> $@; \ - cat $$name.dj >> $@; \ - echo '"###;' >> $@; \ - printf ' let expected = r###"' >> $@; \ - cat $$name.html >> $@; \ - echo '"###;' >> $@; \ - echo " suite_test!(src, expected);" >> $@; \ - echo "}" >> $@; \ - done + skip_reason=$$(grep -E "^$${name_snake}:" skip | cut -d: -f2); \ + [ -n "$$skip_reason" ] && echo "#[ignore = \"$${skip_reason}\"]"; \ + echo "#[test]"; \ + echo "fn $$name_snake() {"; \ + printf ' let src = r###"'; \ + cat $$name.dj; \ + echo '"###;'; \ + printf ' let expected = r###"'; \ + cat $$name.html; \ + echo '"###;'; \ + echo " suite_test!(src, expected);"; \ + echo "}"; \ + done >> $@ html: djot-js ${TEST_DJ} echo ${TEST} diff --git a/tests/bench/skip b/tests/bench/skip new file mode 100644 index 0000000..d9afe9e --- /dev/null +++ b/tests/bench/skip @@ -0,0 +1,3 @@ +block_list_flat:large list marker number +inline_links_flat:escaped attributes, empty hrefs +inline_links_nested:empty link text diff --git a/tests/lib.rs b/tests/lib.rs index 5c72e80..984b610 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -1,5 +1,7 @@ +#[rustfmt::skip] #[cfg(feature = "suite_bench")] mod bench; +#[rustfmt::skip] #[cfg(feature = "suite")] mod suite; diff --git a/tests/suite/gen.awk b/tests/suite/gen.awk index 6d6c699..40e1cdc 100644 --- a/tests/suite/gen.awk +++ b/tests/suite/gen.awk @@ -1,4 +1,6 @@ BEGIN { + FS=":" + while (getline < "skip") skips[$1]=$2 print "use crate::suite_test;" print "" } @@ -7,14 +9,16 @@ $0 ~ "^`{3,}$" { l=length($0) if (fence == 0) { # enter fence print "#[test]" - printf "fn test%02d() {\n", count - printf " let src = r##\"" fence=l - count+=1 } else if (fence == l) { # exit fence if (ignore) { ignore=0 } else { + printf " let expected = r##\"" + close("src") + while (getline l < "src") print l + close("src") + system("rm -f src") print "\"##;" print " suite_test!(src, expected);" print "}" @@ -22,7 +26,7 @@ $0 ~ "^`{3,}$" { } fence=0 } else { - print $0 # md/html + print $0 > "src" # md/html } next } @@ -34,14 +38,25 @@ fence == 0 && $0 ~ "^`{3,} .*$" { } $0 ~ "^\\.$" && !ignore { # enter html + close("src") + cmd="cat src | md5sum | cut -c-7" + cmd | getline hash + close(cmd) + if (hash in skips) printf "#[ignore = \"%s\"]\n", skips[hash] + printf "fn test_%s() {\n", hash + printf " let src = r##\"" + while (getline l < "src") print l + close("src") + system("rm -f src") print "\"##;" - printf " let expected = r##\"" next } !ignore { - if (fence==0 && $0 != "") { # comment - printf "// " + if (fence) { + # write to file so content can be piped to md5sum (without having to shell escape) + print $0 > "src" # md/html + } else if ($0 != "") { + printf "// %s\n", $0 # comment } - print $0 # comment/md/html } diff --git a/tests/suite/skip b/tests/suite/skip new file mode 100644 index 0000000..ee358ac --- /dev/null +++ b/tests/suite/skip @@ -0,0 +1,20 @@ +38d85f9:multi-line block attributes +6c14561:multi-line block attributes +48546bb:escape in attributes +6bc4257:escape in attributes +613a9d6:attribute container precedence +f4f22fc:attribute key class order +ae6fc15:bugged left/right quote +168469a:bugged left/right quote +2056174:unicode whitespace emph +2e8fffa:unicode whitespace strong +e1f5b5e:untrimmed whitespace before linebreak +07888f3:div close within raw block +8423412:heading id conflict with existing id +00a46ed:clear inline formatting from link tags +a8e17c3:empty href +c0a3dec:escape in url +e66af00:url container precedence +61876cf:roman alpha ambiguity +f31b357:roman alpha ambiguity +642d380:table end in verbatim inline