mv suite{_bench} to test-html-{ut,ref} crates

- allow compiling/running html tests without compiling main crate tests
  (useful when e.g. making type changes to events but html unaffected)
- avoid need for future flags in main crate

.github/workflows/ci.yml

@@ -34,8 +34,8 @@ jobs:
             RUSTDOCFLAGS: -D warnings
         run: |
           make check
-  suite:
+  test_html:
-    name: Build and run external tests
+    name: Build and run HTML tests
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout repo"
@@ -44,14 +44,14 @@ jobs:
         run: |
           rustup update 1.56
           rustup default 1.56
-      - name: "Run unit tests"
+      - name: "Run HTML unit tests"
-        run: make suite
+        run: make test_html_ut
       - name: "Setup node"
         uses: actions/setup-node@v3
         with:
           node-version: 18
-      - name: "Compare benchmark files"
+      - name: "Compare HTML with reference implementation"
-        run: make suite_bench
+        run: make test_html_ref
   lint:
     name: Lint
     runs-on: ubuntu-latest

Cargo.lock

@@ -458,6 +458,20 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "test-html-ref"
+version = "0.1.0"
+dependencies = [
+ "jotdown",
+]
+
+[[package]]
+name = "test-html-ut"
+version = "0.1.0"
+dependencies = [
+ "jotdown",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.16.0"

Cargo.toml

@@ -26,6 +26,8 @@ members = [
     "bench/iai",
     "bench/input",
     "examples/jotdown_wasm",
+    "tests/html-ref",
+    "tests/html-ut",
 ]
 exclude = [
     "tests/afl",
@@ -39,6 +41,4 @@ doc = false
 [features]
 default = ["html"]
 html = [] # html renderer and minimal cli binary
-suite = [] # test suite
-suite_bench = [] # bench test suite
 deterministic = [] # for stable fuzzing

Makefile

@@ -25,24 +25,22 @@ check:
 	cargo test --workspace
 	cargo test --workspace --no-default-features
 
-.PHONY: suite
+.PHONY: test_html_ut
-suite:
+test_html_ut:
 	git submodule update --init modules/djot.js
 	for f in $$(find modules/djot.js/test -name '*.test' | xargs basename -a); do \
-		ln -fs ../../modules/djot.js/test/$$f tests/suite/djot_js_$$f; \
+		ln -fs ../../../modules/djot.js/test/$$f tests/html-ut/ut/djot_js_$$f; \
 	done
-	(cd tests/suite && make)
+	cargo test -p test-html-ut
-	cargo test --features suite suite::
 
-.PHONY: suite_bench
+.PHONY: test_html_ref
-suite_bench:
+test_html_ref:
 	git submodule update --init modules/djot.js
 	for f in $$(find modules/djot.js/bench -name '*.dj' | xargs basename -a); do \
 		dst=$$(echo $$f | sed 's/-/_/g'); \
-		ln -fs ../../modules/djot.js/bench/$$f tests/bench/$$dst; \
+		ln -fs ../../modules/djot.js/bench/$$f tests/html-ref/$$dst; \
 	done
-	(cd tests/bench && make)
+	cargo test -p test-html-ref
-	cargo test --features suite_bench bench::
 
 .PHONY: bench
 bench:
@@ -52,9 +50,6 @@ bench:
 		ln -fs ../../modules/djot.js/bench/$$f bench/input/$$dst; \
 	done
 
-cov: suite suite_bench
-	LLVM_COV=llvm-cov LLVM_PROFDATA=llvm-profdata cargo llvm-cov --features=suite,suite_bench --workspace --html --ignore-run-fail
-
 AFL_TARGET?=parse
 AFL_JOBS?=1
 AFL_TARGET_CRASH?=crashes
@@ -103,10 +98,10 @@ clean:
 	cargo clean
 	rm -rf bench/iai/target
 	git submodule deinit -f --all
-	find tests -type l -path 'tests/suite/*.test' -print0 | xargs -0 rm -f
+	find tests -type l -path 'tests/html-ut/ut/*.test' -print0 | xargs -0 rm -f
-	(cd tests/suite && make clean)
+	(cd tests/html-ut && make clean)
-	rm -f tests/bench/*.dj
+	rm -f tests/html-ref/*.dj
-	(cd tests/bench && make clean)
+	(cd tests/html-ref && make clean)
 	find bench -type l -path 'bench/*.dj' -print0 | xargs -0 rm -f
 	rm -rf tests/afl/out
 	(cd examples/jotdown_wasm && make clean)

README.md

@@ -121,19 +121,19 @@ including:
 - footnotes.
 
 The HTML output is in some cases not exactly identical to the [reference
-implementation][djot-js]. There are two test suites that compares Jotdown with
+implementation][djot-js]. There are two test suites that compares Jotdown's
-the reference implementation. One uses the unit tests of the reference
+HTML output with that of the reference implementation. One uses the unit tests
-implementation and runs them with Jotdown. It can be run with:
+of the reference implementation and runs them with Jotdown. It can be run with:
 
 ```
-$ make suite
+$ make test_html_ut
 ```
 
 Another target uses the reference implementation to generate html output for
 its benchmark files and compares it to the output of Jotdown:
 
 ```
-$ make suite_bench
+$ make test_html_ref
 ```
 
 Note that it requires node in order to run the reference implementation.

tests/html-ref/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "test-html-ref"
+description = "Reference implementation HTML output comparison tests"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+jotdown = { path = "../.." }
+
+[lib]
+name = "test_html_ref"
+path = "lib.rs"

tests/html-ref/Makefile

@@ -6,8 +6,8 @@ TEST=${TEST_DJ:.dj=}
 DJOT_JS=../../modules/djot.js
 DJOT_JS_SRC=$(shell find ${DJOT_JS}/src -name '.ts')
 
-mod.rs: ${TEST_DJ} html
+ref.rs: ${TEST_DJ} html
-	echo "use crate::suite_test;" > $@
+	echo "use crate::compare;" > $@
 	for name in ${TEST}; do \
 		name_snake=$$(basename -a $$name); \
 		skip_reason=$$(grep -E "^$${name_snake}:" skip | cut -d: -f2); \
@@ -17,10 +17,8 @@ mod.rs: ${TEST_DJ} html
 		printf '    let src = r###"'; \
 		cat $$name.dj; \
 		echo '"###;'; \
-		printf '    let expected = r###"'; \
+		printf '    let expected = "%s";' "$$name.html"; \
-		cat $$name.html; \
+		echo "    compare!(src, expected);"; \
-		echo '"###;'; \
-		echo "    suite_test!(src, expected);"; \
 		echo "}"; \
 	done >> $@
 
@@ -36,6 +34,7 @@ djot-js: ${DJOT_JS_SRC}
 	chmod +x $@
 
 clean:
-	rm -f *.rs *.html
+	rm -f ref.rs
+	rm -f *.html
 	rm -f html
 	rm -f djot-js

tests/html-ref/build.rs

@@ -0,0 +1,17 @@
+fn main() {
+    let has_dj = std::fs::read_dir(".").unwrap().any(|e| {
+        e.map_or(false, |e| {
+            e.path()
+                .extension()
+                .map_or(false, |ext| ext.to_str() == Some("dj"))
+        })
+    });
+    if has_dj {
+        let status = std::process::Command::new("make")
+            .status()
+            .expect("failed to execute make");
+        assert!(status.success());
+    } else {
+        std::fs::write("ref.rs", &[b'\n']).unwrap();
+    }
+}

tests/html-ref/lib.rs

@@ -0,0 +1,32 @@
+#[cfg(test)]
+mod r#ref;
+
+#[macro_export]
+macro_rules! compare {
+    ($src:expr, $expected:expr) => {
+        use jotdown::Render;
+        let src = $src;
+        let expected = std::fs::read_to_string($expected).expect("read failed");
+        let p = jotdown::Parser::new(src);
+        let mut actual = String::new();
+        jotdown::html::Renderer::default()
+            .push(p, &mut actual)
+            .unwrap();
+        assert_eq!(actual, expected, "\n{}", {
+            use std::io::Write;
+            let mut child = std::process::Command::new("diff")
+                .arg("--color=always")
+                .arg("-")
+                .arg($expected)
+                .stdin(std::process::Stdio::piped())
+                .stdout(std::process::Stdio::piped())
+                .spawn()
+                .expect("spawn diff failed");
+            let mut stdin = child.stdin.take().unwrap();
+            let actual = actual.clone();
+            std::thread::spawn(move || stdin.write_all(actual.as_bytes()).unwrap());
+            let stdout = child.wait_with_output().unwrap().stdout;
+            String::from_utf8(stdout).unwrap()
+        });
+    };
+}

tests/html-ut/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "test-html-ut"
+description = "HTML output unit tests."
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+jotdown = { path = "../.." }
+
+[lib]
+name = "test_html_ut"
+path = "lib.rs"

tests/html-ut/Makefile

@@ -2,25 +2,23 @@
 
 .SUFFIXES: .test .rs
 
-TEST=$(shell find . -name '*.test' | sort)
+TEST=$(shell find ut -name '*.test' | sort)
 TEST_RS=${TEST:.test=.rs}
 
 BLACKLIST += djot_js_filters # lua filters not implemented
 BLACKLIST += djot_js_symb # uses ast
 BLACKLIST += djot_js_sourcepos # not parsable
 
-.PHONY: suite
+ut/mod.rs: ${TEST_RS}
-suite: mod.rs
+	mkdir -p ut
-
+	rm -f $@
-mod.rs: ${TEST_RS}
-	printf "" > $@
 	for f in ${TEST}; do \
 		name=$$(basename -s .test $$f); \
 		echo ${BLACKLIST} | tr ' ' '\n' | grep -q $$name || echo "mod $$name;" >> $@; \
 	done
 
 .test.rs:
-	gawk -fgen.awk $< > $@
+	gawk -fgen.awk $< | head -n-1 > $@
 
 clean:
-	rm -f *.rs
+	rm -f ut/*.rs

tests/html-ut/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    let status = std::process::Command::new("make")
+        .status()
+        .expect("failed to execute make");
+    assert!(status.success());
+}

tests/html-ut/cmp.rs

@@ -0,0 +1,47 @@
+#[macro_export]
+macro_rules! compare {
+    ($src:expr, $expected:expr) => {
+        use jotdown::Render;
+        let src = $src;
+        let expected = $expected;
+        let p = jotdown::Parser::new(src);
+        let mut actual = String::new();
+        jotdown::html::Renderer::default()
+            .push(p, &mut actual)
+            .unwrap();
+        assert_eq!(
+            actual.trim(),
+            expected.trim(),
+            concat!(
+                "\n",
+                "\x1b[0;1m========================= INPUT ============================\x1b[0m\n",
+                "\x1b[2m{}",
+                "\x1b[0;1m=================== ACTUAL vs EXPECTED =====================\x1b[0m\n",
+                "{}",
+                "\x1b[0;1m============================================================\x1b[0m\n",
+            ),
+            $src,
+            {
+                let a = actual.trim().split('\n');
+                let b = expected.trim().split('\n');
+                let max = a.clone().count().max(b.clone().count());
+                let a_width = a.clone().map(|a| a.len()).max().unwrap_or(0);
+                a.chain(std::iter::repeat(""))
+                    .zip(b.chain(std::iter::repeat("")))
+                    .take(max)
+                    .map(|(a, b)| {
+                        format!(
+                            "\x1b[{}m{:a_width$}\x1b[0m    {}=    \x1b[{}m{}\x1b[0m\n",
+                            if a == b { "2" } else { "31" },
+                            a,
+                            if a == b { '=' } else { '!' },
+                            if a == b { "2" } else { "32" },
+                            b,
+                            a_width = a_width,
+                        )
+                    })
+                    .collect::<String>()
+            },
+        );
+    };
+}

tests/html-ut/gen.awk

@@ -1,7 +1,7 @@
 BEGIN {
     FS=":"
     while (getline < "skip") skips[$1]=$2
-    print "use crate::suite_test;"
+    print "use crate::compare;"
     print ""
 }
 
@@ -20,7 +20,7 @@ $0 ~ "^`{3,}$" {
             close("src")
             system("rm -f src")
             print "\"##;"
-            print "    suite_test!(src, expected);"
+            print "    compare!(src, expected);"
             print "}"
             print ""
         }

tests/html-ut/lib.rs

@@ -1,13 +1,8 @@
-#[rustfmt::skip]
+#[cfg(test)]
-#[cfg(feature = "suite_bench")]
+mod ut;
-mod bench;
-#[rustfmt::skip]
-#[cfg(feature = "suite")]
-mod suite;
 
-#[cfg(any(feature = "suite", feature = "suite_bench"))]
 #[macro_export]
-macro_rules! suite_test {
+macro_rules! compare {
     ($src:expr, $expected:expr) => {
         use jotdown::Render;
         let src = $src;