PR #13 Setup CI

closes #9
closes #13

.github/workflows/ci.yml

@@ -0,0 +1,80 @@
+name: ci
+
+on:
+  pull_request:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  test:
+    name: Build and run tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        toolchain:
+          - "1.56"
+          - stable
+    steps:
+      - name: "Checkout repo"
+        uses: actions/checkout@v3
+      - name: "Setup toolchain"
+        run: |
+          rustup update ${{ matrix.toolchain }}
+          rustup default ${{ matrix.toolchain }}
+      - name: "Build"
+        run: |
+          make all
+          cargo build --workspace --no-default-features
+      - name: "Run tests"
+        env:
+            RUSTDOCFLAGS: -D warnings
+        run: |
+          make check
+  suite:
+    name: Build and run external tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout repo"
+        uses: actions/checkout@v3
+      - name: "Setup toolchain"
+        run: |
+          rustup update 1.56
+          rustup default 1.56
+      - name: "Run unit tests"
+        run: make suite
+      - name: "Setup node"
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: "Compare benchmark files"
+        run: make suite_bench
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v3
+      - name: "Setup toolchain"
+        run: |
+          rustup update 1.56
+          rustup default 1.56
+          rustup component add rustfmt
+          rustup component add clippy
+      - name: "Check linting"
+        run: make lint
+  fuzz:
+    name: Fuzz
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v3
+      - name: "Setup toolchain"
+        run: |
+          rustup update nightly
+          rustup default nightly
+          cargo install afl
+      - name: "Fuzz"
+        run: |
+          echo core | sudo tee /proc/sys/kernel/core_pattern
+          make afl_quick

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "jgm/djot.js"]
     path = modules/djot.js
-    url = git@github.com:jgm/djot.js.git
+    url = https://github.com/jgm/djot.js.git

Cargo.lock

@@ -2,130 +2,22 @@
 # It is not intended for manual editing.
 version = 3
 
-[[package]]
-name = "afl"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c0f4180b6d7095a1c3130f1aadead4ece7fb3094dc724c701adb30a92a95228"
-dependencies = [
- "cc",
- "clap",
- "libc",
- "rustc_version",
- "xdg",
-]
-
-[[package]]
-name = "ansi_term"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi",
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
 [[package]]
 name = "bumpalo"
 version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
-[[package]]
-name = "cc"
-version = "1.0.79"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "clap"
-version = "2.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
-dependencies = [
- "ansi_term",
- "atty",
- "bitflags",
- "strsim",
- "textwrap",
- "unicode-width",
- "vec_map",
-]
-
-[[package]]
-name = "dirs"
-version = "4.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
-dependencies = [
- "dirs-sys",
-]
-
-[[package]]
-name = "dirs-sys"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "jotdown"
 version = "0.1.0"
 
-[[package]]
-name = "jotdown-afl"
-version = "0.1.0"
-dependencies = [
- "afl",
- "jotdown",
-]
-
 [[package]]
 name = "jotdown_wasm"
 version = "0.1.0"
@@ -144,12 +36,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "libc"
-version = "0.2.139"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
-
 [[package]]
 name = "log"
 version = "0.4.17"
@@ -183,47 +69,6 @@ dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "redox_users"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
-dependencies = [
- "getrandom",
- "redox_syscall",
- "thiserror",
-]
-
-[[package]]
-name = "rustc_version"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
-dependencies = [
- "semver",
-]
-
-[[package]]
-name = "semver"
-version = "1.0.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
-
-[[package]]
-name = "strsim"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
-
 [[package]]
 name = "syn"
 version = "1.0.107"
@@ -235,59 +80,12 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
-[[package]]
-name = "thiserror"
-version = "1.0.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
-[[package]]
-name = "unicode-width"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
-
-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.84"
@@ -351,34 +149,3 @@ dependencies = [
  "js-sys",
  "wasm-bindgen",
 ]
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "xdg"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c4583db5cbd4c4c0303df2d15af80f0539db703fa1c68802d4cbbd2dd0f88f6"
-dependencies = [
- "dirs",
-]

Cargo.toml

@@ -21,6 +21,8 @@ exclude = [
 [workspace]
 members = [
     "examples/jotdown_wasm",
+]
+exclude = [
     "tests/afl",
 ]
 

Makefile

@@ -1,5 +1,30 @@
 .POSIX:
 
+all: jotdown docs
+	cargo build --workspace
+
+jotdown: target/release/jotdown
+	cp $< $@
+
+target/release/jotdown:
+	cargo build --release
+
+.PHONY:
+docs:
+	cargo doc --no-deps --workspace
+
+.PHONY: lint
+lint:
+	cargo fmt --all -- --check
+	cargo clippy -- -D warnings
+	cargo clippy --no-default-features -- -D warnings
+	cargo clippy --all-features -- -D warnings
+
+.PHONY: check
+check:
+	cargo test --workspace
+	cargo test --workspace --no-default-features
+
 .PHONY: suite
 suite:
 	git submodule update --init modules/djot.js
@@ -36,13 +61,20 @@ afl:
 	rm -rf tests/afl/out
 	(cd tests/afl && \
 		cargo afl build --release --config profile.release.debug-assertions=true && \
-		(AFL_NO_UI=1 cargo afl fuzz -i in -o out -Mm ../../target/release/${AFL_TARGET} &) && \
+		(AFL_NO_UI=1 cargo afl fuzz -i in -o out -Mm target/release/${AFL_TARGET} &) && \
 		for i in $$(seq $$((${AFL_JOBS} - 1))); do \
-			AFL_NO_UI=1 cargo afl fuzz -i in -o out -Ss$$i ../../target/release/${AFL_TARGET} & \
+			AFL_NO_UI=1 cargo afl fuzz -i in -o out -Ss$$i target/release/${AFL_TARGET} & \
 		done; \
 		trap - EXIT;\
 		cat) # keep process alive for trap
 
+afl_quick:
+	rm -rf tests/afl/out
+	(cd tests/afl && \
+		cargo afl build --release --config profile.release.debug-assertions=true && \
+		AFL_NO_UI=1 AFL_BENCH_UNTIL_CRASH=1 \
+			cargo afl fuzz -i in -o out -V 60 target/release/${AFL_TARGET})
+
 afl_crash:
 	set +e; \
 	for f in $$(find tests/afl/out -path '*/${AFL_TARGET_CRASH}/id*'); do \

src/attr.rs

@@ -27,6 +27,7 @@ pub fn valid<I: Iterator<Item = char>>(chars: I) -> (usize, bool) {
 /// A collection of attributes, i.e. a key-value map.
 // Attributes are relatively rare, we choose to pay 8 bytes always and sometimes an extra
 // indirection instead of always 24 bytes.
+#[allow(clippy::box_vec)]
 #[derive(Debug, Clone, PartialEq, Eq, Default)]
 pub struct Attributes<'s>(Option<Box<Vec<(&'s str, CowStr<'s>)>>>);
 

src/block.rs

@@ -351,13 +351,13 @@ impl<'s> TreeParser<'s> {
         });
 
         if let ListItem(ty) = c {
-            if self
+            let same_depth = self
                 .open_lists
                 .last()
                 .map_or(true, |OpenList { depth, .. }| {
                     usize::from(*depth) < self.tree.depth()
-                })
+                });
-            {
+            if same_depth {
                 let tight = true;
                 let node = self.tree.enter(
                     Node::Container(Container::List(ListKind { ty, tight })),

src/inline.rs

@@ -442,11 +442,10 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
                     if matches!(dir, Dir::Open) {
                         return None;
                     }
-                    if matches!(dir, Dir::Both)
+                    let whitespace_after = self.events.back().map_or(false, |ev| {
-                        && self.events.back().map_or(false, |ev| {
                         matches!(ev.kind, EventKind::Whitespace | EventKind::Atom(Softbreak))
-                        })
+                    });
-                    {
+                    if matches!(dir, Dir::Both) && whitespace_after {
                         return None;
                     }
 
@@ -791,6 +790,7 @@ impl<I: Iterator<Item = char> + Clone> Iterator for Parser<I> {
     type Item = Event;
 
     fn next(&mut self) -> Option<Self::Item> {
+        #[allow(clippy::blocks_in_if_conditions)]
         while self.events.is_empty()
             || !self.openers.is_empty()
             || self // for merge or attributes
@@ -813,12 +813,13 @@ impl<I: Iterator<Item = char> + Clone> Iterator for Parser<I> {
                 EventKind::Str | EventKind::Whitespace => {
                     // merge str events
                     let mut span = e.span;
-                    while self.events.front().map_or(false, |e| {
+                    let should_merge = |e: &Event, span: Span| {
                         matches!(
                             e.kind,
                             EventKind::Str | EventKind::Whitespace | EventKind::Placeholder
                         ) && span.end() == e.span.start()
-                    }) {
+                    };
+                    while self.events.front().map_or(false, |e| should_merge(e, span)) {
                         let ev = self.events.pop_front().unwrap();
                         span = span.union(ev.span);
                     }

src/lib.rs

@@ -241,7 +241,8 @@ pub enum Container<'s> {
 
 impl<'s> Container<'s> {
     /// Is a block element.
-    fn is_block(&self) -> bool {
+    #[must_use]
+    pub fn is_block(&self) -> bool {
         match self {
             Self::Blockquote
             | Self::List { .. }
@@ -278,7 +279,8 @@ impl<'s> Container<'s> {
     }
 
     /// Is a block element that may contain children blocks.
-    fn is_block_container(&self) -> bool {
+    #[must_use]
+    pub fn is_block_container(&self) -> bool {
         match self {
             Self::Blockquote
             | Self::List { .. }

tests/bench/Makefile

@@ -10,17 +10,19 @@ mod.rs: ${TEST_DJ} html
 	echo "use crate::suite_test;" > $@
 	for name in ${TEST}; do \
 		name_snake=$$(basename -a $$name | sed 's/-/_/g'); \
-		echo "#[test]" >> $@; \
+		skip_reason=$$(grep -E "^$${name_snake}:" skip | cut -d: -f2); \
-		echo "fn test_$$name_snake() {" >> $@; \
+		[ -n "$$skip_reason" ] && echo "#[ignore = \"$${skip_reason}\"]"; \
-		printf '    let src = r###"' >> $@; \
+		echo "#[test]"; \
-		cat $$name.dj >> $@; \
+		echo "fn $$name_snake() {"; \
-		echo '"###;' >> $@; \
+		printf '    let src = r###"'; \
-		printf '    let expected = r###"' >> $@; \
+		cat $$name.dj; \
-		cat $$name.html >> $@; \
+		echo '"###;'; \
-		echo '"###;' >> $@; \
+		printf '    let expected = r###"'; \
-		echo "    suite_test!(src, expected);" >> $@; \
+		cat $$name.html; \
-		echo "}" >> $@; \
+		echo '"###;'; \
-	done
+		echo "    suite_test!(src, expected);"; \
+		echo "}"; \
+	done >> $@
 
 html: djot-js ${TEST_DJ}
 	echo ${TEST}

tests/bench/skip

@@ -0,0 +1,3 @@
+block_list_flat:large list marker number
+inline_links_flat:escaped attributes, empty hrefs
+inline_links_nested:empty link text

tests/lib.rs

@@ -1,5 +1,7 @@
+#[rustfmt::skip]
 #[cfg(feature = "suite_bench")]
 mod bench;
+#[rustfmt::skip]
 #[cfg(feature = "suite")]
 mod suite;
 

tests/suite/gen.awk

@@ -1,4 +1,6 @@
 BEGIN {
+    FS=":"
+    while (getline < "skip") skips[$1]=$2
     print "use crate::suite_test;"
     print ""
 }
@@ -7,14 +9,16 @@ $0 ~ "^`{3,}$" {
     l=length($0)
     if (fence == 0) { # enter fence
         print "#[test]"
-        printf "fn test%02d() {\n", count
-        printf "    let src = r##\""
         fence=l
-        count+=1
     } else if (fence == l) { # exit fence
         if (ignore) {
             ignore=0
         } else {
+            printf "    let expected = r##\""
+            close("src")
+            while (getline l < "src") print l
+            close("src")
+            system("rm -f src")
             print "\"##;"
             print "    suite_test!(src, expected);"
             print "}"
@@ -22,7 +26,7 @@ $0 ~ "^`{3,}$" {
         }
         fence=0
     } else {
-        print $0 # md/html
+        print $0 > "src" # md/html
     }
     next
 }
@@ -34,14 +38,25 @@ fence == 0 && $0 ~ "^`{3,} .*$" {
 }
 
 $0 ~ "^\\.$" && !ignore { # enter html
+    close("src")
+    cmd="cat src | md5sum | cut -c-7"
+    cmd | getline hash
+    close(cmd)
+    if (hash in skips) printf "#[ignore = \"%s\"]\n", skips[hash]
+    printf "fn test_%s() {\n", hash
+    printf "    let src = r##\""
+    while (getline l < "src") print l
+    close("src")
+    system("rm -f src")
     print "\"##;"
-    printf "    let expected = r##\""
     next
 }
 
 !ignore {
-    if (fence==0 && $0 != "") { # comment
+    if (fence) {
-        printf "// "
+        # write to file so content can be piped to md5sum (without having to shell escape)
+        print $0 > "src" # md/html
+    } else if ($0 != "") {
+        printf "// %s\n", $0 # comment
     }
-    print $0 # comment/md/html
 }

tests/suite/skip

@@ -0,0 +1,20 @@
+38d85f9:multi-line block attributes
+6c14561:multi-line block attributes
+48546bb:escape in attributes
+6bc4257:escape in attributes
+613a9d6:attribute container precedence
+f4f22fc:attribute key class order
+ae6fc15:bugged left/right quote
+168469a:bugged left/right quote
+2056174:unicode whitespace emph
+2e8fffa:unicode whitespace strong
+e1f5b5e:untrimmed whitespace before linebreak
+07888f3:div close within raw block
+8423412:heading id conflict with existing id
+00a46ed:clear inline formatting from link tags
+a8e17c3:empty href
+c0a3dec:escape in url
+e66af00:url container precedence
+61876cf:roman alpha ambiguity
+f31b357:roman alpha ambiguity
+642d380:table end in verbatim inline