diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d02c622..55571f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,8 +34,8 @@ jobs: RUSTDOCFLAGS: -D warnings run: | make check - suite: - name: Build and run external tests + test_html: + name: Build and run HTML tests runs-on: ubuntu-latest steps: - name: "Checkout repo" @@ -44,14 +44,14 @@ jobs: run: | rustup update 1.56 rustup default 1.56 - - name: "Run unit tests" - run: make suite + - name: "Run HTML unit tests" + run: make test_html_ut - name: "Setup node" uses: actions/setup-node@v3 with: node-version: 18 - - name: "Compare benchmark files" - run: make suite_bench + - name: "Compare HTML with reference implementation" + run: make test_html_ref lint: name: Lint runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 08690ae..5232ccb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -458,6 +458,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "test-html-ref" +version = "0.1.0" +dependencies = [ + "jotdown", +] + +[[package]] +name = "test-html-ut" +version = "0.1.0" +dependencies = [ + "jotdown", +] + [[package]] name = "textwrap" version = "0.16.0" diff --git a/Cargo.toml b/Cargo.toml index aa64f97..53604af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ members = [ "bench/iai", "bench/input", "examples/jotdown_wasm", + "tests/html-ref", + "tests/html-ut", ] exclude = [ "tests/afl", @@ -39,6 +41,4 @@ doc = false [features] default = ["html"] html = [] # html renderer and minimal cli binary -suite = [] # test suite -suite_bench = [] # bench test suite deterministic = [] # for stable fuzzing diff --git a/Makefile b/Makefile index 79c38bb..a5e4632 100644 --- a/Makefile +++ b/Makefile @@ -25,24 +25,22 @@ check: cargo test --workspace cargo test --workspace --no-default-features -.PHONY: suite -suite: +.PHONY: test_html_ut +test_html_ut: git submodule update --init modules/djot.js for f in $$(find modules/djot.js/test -name '*.test' | xargs basename -a); do \ - ln -fs ../../modules/djot.js/test/$$f tests/suite/djot_js_$$f; \ + ln -fs ../../../modules/djot.js/test/$$f tests/html-ut/ut/djot_js_$$f; \ done - (cd tests/suite && make) - cargo test --features suite suite:: + cargo test -p test-html-ut -.PHONY: suite_bench -suite_bench: +.PHONY: test_html_ref +test_html_ref: git submodule update --init modules/djot.js for f in $$(find modules/djot.js/bench -name '*.dj' | xargs basename -a); do \ dst=$$(echo $$f | sed 's/-/_/g'); \ - ln -fs ../../modules/djot.js/bench/$$f tests/bench/$$dst; \ + ln -fs ../../modules/djot.js/bench/$$f tests/html-ref/$$dst; \ done - (cd tests/bench && make) - cargo test --features suite_bench bench:: + cargo test -p test-html-ref .PHONY: bench bench: @@ -52,9 +50,6 @@ bench: ln -fs ../../modules/djot.js/bench/$$f bench/input/$$dst; \ done -cov: suite suite_bench - LLVM_COV=llvm-cov LLVM_PROFDATA=llvm-profdata cargo llvm-cov --features=suite,suite_bench --workspace --html --ignore-run-fail - AFL_TARGET?=parse AFL_JOBS?=1 AFL_TARGET_CRASH?=crashes @@ -103,10 +98,10 @@ clean: cargo clean rm -rf bench/iai/target git submodule deinit -f --all - find tests -type l -path 'tests/suite/*.test' -print0 | xargs -0 rm -f - (cd tests/suite && make clean) - rm -f tests/bench/*.dj - (cd tests/bench && make clean) + find tests -type l -path 'tests/html-ut/ut/*.test' -print0 | xargs -0 rm -f + (cd tests/html-ut && make clean) + rm -f tests/html-ref/*.dj + (cd tests/html-ref && make clean) find bench -type l -path 'bench/*.dj' -print0 | xargs -0 rm -f rm -rf tests/afl/out (cd examples/jotdown_wasm && make clean) diff --git a/README.md b/README.md index 10f1e35..b0a96e2 100644 --- a/README.md +++ b/README.md @@ -121,19 +121,19 @@ including: - footnotes. The HTML output is in some cases not exactly identical to the [reference -implementation][djot-js]. There are two test suites that compares Jotdown with -the reference implementation. One uses the unit tests of the reference -implementation and runs them with Jotdown. It can be run with: +implementation][djot-js]. There are two test suites that compares Jotdown's +HTML output with that of the reference implementation. One uses the unit tests +of the reference implementation and runs them with Jotdown. It can be run with: ``` -$ make suite +$ make test_html_ut ``` Another target uses the reference implementation to generate html output for its benchmark files and compares it to the output of Jotdown: ``` -$ make suite_bench +$ make test_html_ref ``` Note that it requires node in order to run the reference implementation. diff --git a/tests/html-ref/Cargo.toml b/tests/html-ref/Cargo.toml new file mode 100644 index 0000000..b46b4cc --- /dev/null +++ b/tests/html-ref/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "test-html-ref" +description = "Reference implementation HTML output comparison tests" +version = "0.1.0" +edition = "2021" + +[dependencies] +jotdown = { path = "../.." } + +[lib] +name = "test_html_ref" +path = "lib.rs" diff --git a/tests/bench/Makefile b/tests/html-ref/Makefile similarity index 80% rename from tests/bench/Makefile rename to tests/html-ref/Makefile index b84439c..31f416e 100644 --- a/tests/bench/Makefile +++ b/tests/html-ref/Makefile @@ -6,8 +6,8 @@ TEST=${TEST_DJ:.dj=} DJOT_JS=../../modules/djot.js DJOT_JS_SRC=$(shell find ${DJOT_JS}/src -name '.ts') -mod.rs: ${TEST_DJ} html - echo "use crate::suite_test;" > $@ +ref.rs: ${TEST_DJ} html + echo "use crate::compare;" > $@ for name in ${TEST}; do \ name_snake=$$(basename -a $$name); \ skip_reason=$$(grep -E "^$${name_snake}:" skip | cut -d: -f2); \ @@ -17,10 +17,8 @@ mod.rs: ${TEST_DJ} html printf ' let src = r###"'; \ cat $$name.dj; \ echo '"###;'; \ - printf ' let expected = r###"'; \ - cat $$name.html; \ - echo '"###;'; \ - echo " suite_test!(src, expected);"; \ + printf ' let expected = "%s";' "$$name.html"; \ + echo " compare!(src, expected);"; \ echo "}"; \ done >> $@ @@ -36,6 +34,7 @@ djot-js: ${DJOT_JS_SRC} chmod +x $@ clean: - rm -f *.rs *.html + rm -f ref.rs + rm -f *.html rm -f html rm -f djot-js diff --git a/tests/html-ref/build.rs b/tests/html-ref/build.rs new file mode 100644 index 0000000..bccf671 --- /dev/null +++ b/tests/html-ref/build.rs @@ -0,0 +1,17 @@ +fn main() { + let has_dj = std::fs::read_dir(".").unwrap().any(|e| { + e.map_or(false, |e| { + e.path() + .extension() + .map_or(false, |ext| ext.to_str() == Some("dj")) + }) + }); + if has_dj { + let status = std::process::Command::new("make") + .status() + .expect("failed to execute make"); + assert!(status.success()); + } else { + std::fs::write("ref.rs", &[b'\n']).unwrap(); + } +} diff --git a/tests/html-ref/lib.rs b/tests/html-ref/lib.rs new file mode 100644 index 0000000..fabada5 --- /dev/null +++ b/tests/html-ref/lib.rs @@ -0,0 +1,32 @@ +#[cfg(test)] +mod r#ref; + +#[macro_export] +macro_rules! compare { + ($src:expr, $expected:expr) => { + use jotdown::Render; + let src = $src; + let expected = std::fs::read_to_string($expected).expect("read failed"); + let p = jotdown::Parser::new(src); + let mut actual = String::new(); + jotdown::html::Renderer::default() + .push(p, &mut actual) + .unwrap(); + assert_eq!(actual, expected, "\n{}", { + use std::io::Write; + let mut child = std::process::Command::new("diff") + .arg("--color=always") + .arg("-") + .arg($expected) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn diff failed"); + let mut stdin = child.stdin.take().unwrap(); + let actual = actual.clone(); + std::thread::spawn(move || stdin.write_all(actual.as_bytes()).unwrap()); + let stdout = child.wait_with_output().unwrap().stdout; + String::from_utf8(stdout).unwrap() + }); + }; +} diff --git a/tests/bench/skip b/tests/html-ref/skip similarity index 100% rename from tests/bench/skip rename to tests/html-ref/skip diff --git a/tests/html-ut/Cargo.toml b/tests/html-ut/Cargo.toml new file mode 100644 index 0000000..f372f86 --- /dev/null +++ b/tests/html-ut/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "test-html-ut" +description = "HTML output unit tests." +version = "0.1.0" +edition = "2021" + +[dependencies] +jotdown = { path = "../.." } + +[lib] +name = "test_html_ut" +path = "lib.rs" diff --git a/tests/suite/Makefile b/tests/html-ut/Makefile similarity index 72% rename from tests/suite/Makefile rename to tests/html-ut/Makefile index e52b030..d527ba5 100644 --- a/tests/suite/Makefile +++ b/tests/html-ut/Makefile @@ -2,25 +2,23 @@ .SUFFIXES: .test .rs -TEST=$(shell find . -name '*.test' | sort) +TEST=$(shell find ut -name '*.test' | sort) TEST_RS=${TEST:.test=.rs} BLACKLIST += djot_js_filters # lua filters not implemented BLACKLIST += djot_js_symb # uses ast BLACKLIST += djot_js_sourcepos # not parsable -.PHONY: suite -suite: mod.rs - -mod.rs: ${TEST_RS} - printf "" > $@ +ut/mod.rs: ${TEST_RS} + mkdir -p ut + rm -f $@ for f in ${TEST}; do \ name=$$(basename -s .test $$f); \ echo ${BLACKLIST} | tr ' ' '\n' | grep -q $$name || echo "mod $$name;" >> $@; \ done .test.rs: - gawk -fgen.awk $< > $@ + gawk -fgen.awk $< | head -n-1 > $@ clean: - rm -f *.rs + rm -f ut/*.rs diff --git a/tests/html-ut/build.rs b/tests/html-ut/build.rs new file mode 100644 index 0000000..bfbbae2 --- /dev/null +++ b/tests/html-ut/build.rs @@ -0,0 +1,6 @@ +fn main() { + let status = std::process::Command::new("make") + .status() + .expect("failed to execute make"); + assert!(status.success()); +} diff --git a/tests/html-ut/cmp.rs b/tests/html-ut/cmp.rs new file mode 100644 index 0000000..949881f --- /dev/null +++ b/tests/html-ut/cmp.rs @@ -0,0 +1,47 @@ +#[macro_export] +macro_rules! compare { + ($src:expr, $expected:expr) => { + use jotdown::Render; + let src = $src; + let expected = $expected; + let p = jotdown::Parser::new(src); + let mut actual = String::new(); + jotdown::html::Renderer::default() + .push(p, &mut actual) + .unwrap(); + assert_eq!( + actual.trim(), + expected.trim(), + concat!( + "\n", + "\x1b[0;1m========================= INPUT ============================\x1b[0m\n", + "\x1b[2m{}", + "\x1b[0;1m=================== ACTUAL vs EXPECTED =====================\x1b[0m\n", + "{}", + "\x1b[0;1m============================================================\x1b[0m\n", + ), + $src, + { + let a = actual.trim().split('\n'); + let b = expected.trim().split('\n'); + let max = a.clone().count().max(b.clone().count()); + let a_width = a.clone().map(|a| a.len()).max().unwrap_or(0); + a.chain(std::iter::repeat("")) + .zip(b.chain(std::iter::repeat(""))) + .take(max) + .map(|(a, b)| { + format!( + "\x1b[{}m{:a_width$}\x1b[0m {}= \x1b[{}m{}\x1b[0m\n", + if a == b { "2" } else { "31" }, + a, + if a == b { '=' } else { '!' }, + if a == b { "2" } else { "32" }, + b, + a_width = a_width, + ) + }) + .collect::() + }, + ); + }; +} diff --git a/tests/suite/gen.awk b/tests/html-ut/gen.awk similarity index 93% rename from tests/suite/gen.awk rename to tests/html-ut/gen.awk index 40e1cdc..37a6f8c 100644 --- a/tests/suite/gen.awk +++ b/tests/html-ut/gen.awk @@ -1,7 +1,7 @@ BEGIN { FS=":" while (getline < "skip") skips[$1]=$2 - print "use crate::suite_test;" + print "use crate::compare;" print "" } @@ -20,7 +20,7 @@ $0 ~ "^`{3,}$" { close("src") system("rm -f src") print "\"##;" - print " suite_test!(src, expected);" + print " compare!(src, expected);" print "}" print "" } diff --git a/tests/lib.rs b/tests/html-ut/lib.rs similarity index 90% rename from tests/lib.rs rename to tests/html-ut/lib.rs index 4fd36af..ca8943b 100644 --- a/tests/lib.rs +++ b/tests/html-ut/lib.rs @@ -1,13 +1,8 @@ -#[rustfmt::skip] -#[cfg(feature = "suite_bench")] -mod bench; -#[rustfmt::skip] -#[cfg(feature = "suite")] -mod suite; +#[cfg(test)] +mod ut; -#[cfg(any(feature = "suite", feature = "suite_bench"))] #[macro_export] -macro_rules! suite_test { +macro_rules! compare { ($src:expr, $expected:expr) => { use jotdown::Render; let src = $src; diff --git a/tests/suite/skip b/tests/html-ut/skip similarity index 100% rename from tests/suite/skip rename to tests/html-ut/skip diff --git a/tests/suite/footnotes.test b/tests/html-ut/ut/footnotes.test similarity index 100% rename from tests/suite/footnotes.test rename to tests/html-ut/ut/footnotes.test