mv suite{_bench} to test-html-{ut,ref} crates

- allow compiling/running html tests without compiling main crate tests
  (useful when e.g. making type changes to events but html unaffected)
- avoid need for future flags in main crate
This commit is contained in:
Noah Hellman 2023-04-28 20:58:08 +02:00
parent d2a46663f1
commit 3cea79a122
18 changed files with 182 additions and 55 deletions

View file

@ -34,8 +34,8 @@ jobs:
RUSTDOCFLAGS: -D warnings RUSTDOCFLAGS: -D warnings
run: | run: |
make check make check
suite: test_html:
name: Build and run external tests name: Build and run HTML tests
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: "Checkout repo" - name: "Checkout repo"
@ -44,14 +44,14 @@ jobs:
run: | run: |
rustup update 1.56 rustup update 1.56
rustup default 1.56 rustup default 1.56
- name: "Run unit tests" - name: "Run HTML unit tests"
run: make suite run: make test_html_ut
- name: "Setup node" - name: "Setup node"
uses: actions/setup-node@v3 uses: actions/setup-node@v3
with: with:
node-version: 18 node-version: 18
- name: "Compare benchmark files" - name: "Compare HTML with reference implementation"
run: make suite_bench run: make test_html_ref
lint: lint:
name: Lint name: Lint
runs-on: ubuntu-latest runs-on: ubuntu-latest

14
Cargo.lock generated
View file

@ -458,6 +458,20 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "test-html-ref"
version = "0.1.0"
dependencies = [
"jotdown",
]
[[package]]
name = "test-html-ut"
version = "0.1.0"
dependencies = [
"jotdown",
]
[[package]] [[package]]
name = "textwrap" name = "textwrap"
version = "0.16.0" version = "0.16.0"

View file

@ -26,6 +26,8 @@ members = [
"bench/iai", "bench/iai",
"bench/input", "bench/input",
"examples/jotdown_wasm", "examples/jotdown_wasm",
"tests/html-ref",
"tests/html-ut",
] ]
exclude = [ exclude = [
"tests/afl", "tests/afl",
@ -39,6 +41,4 @@ doc = false
[features] [features]
default = ["html"] default = ["html"]
html = [] # html renderer and minimal cli binary html = [] # html renderer and minimal cli binary
suite = [] # test suite
suite_bench = [] # bench test suite
deterministic = [] # for stable fuzzing deterministic = [] # for stable fuzzing

View file

@ -25,24 +25,22 @@ check:
cargo test --workspace cargo test --workspace
cargo test --workspace --no-default-features cargo test --workspace --no-default-features
.PHONY: suite .PHONY: test_html_ut
suite: test_html_ut:
git submodule update --init modules/djot.js git submodule update --init modules/djot.js
for f in $$(find modules/djot.js/test -name '*.test' | xargs basename -a); do \ for f in $$(find modules/djot.js/test -name '*.test' | xargs basename -a); do \
ln -fs ../../modules/djot.js/test/$$f tests/suite/djot_js_$$f; \ ln -fs ../../../modules/djot.js/test/$$f tests/html-ut/ut/djot_js_$$f; \
done done
(cd tests/suite && make) cargo test -p test-html-ut
cargo test --features suite suite::
.PHONY: suite_bench .PHONY: test_html_ref
suite_bench: test_html_ref:
git submodule update --init modules/djot.js git submodule update --init modules/djot.js
for f in $$(find modules/djot.js/bench -name '*.dj' | xargs basename -a); do \ for f in $$(find modules/djot.js/bench -name '*.dj' | xargs basename -a); do \
dst=$$(echo $$f | sed 's/-/_/g'); \ dst=$$(echo $$f | sed 's/-/_/g'); \
ln -fs ../../modules/djot.js/bench/$$f tests/bench/$$dst; \ ln -fs ../../modules/djot.js/bench/$$f tests/html-ref/$$dst; \
done done
(cd tests/bench && make) cargo test -p test-html-ref
cargo test --features suite_bench bench::
.PHONY: bench .PHONY: bench
bench: bench:
@ -52,9 +50,6 @@ bench:
ln -fs ../../modules/djot.js/bench/$$f bench/input/$$dst; \ ln -fs ../../modules/djot.js/bench/$$f bench/input/$$dst; \
done done
cov: suite suite_bench
LLVM_COV=llvm-cov LLVM_PROFDATA=llvm-profdata cargo llvm-cov --features=suite,suite_bench --workspace --html --ignore-run-fail
AFL_TARGET?=parse AFL_TARGET?=parse
AFL_JOBS?=1 AFL_JOBS?=1
AFL_TARGET_CRASH?=crashes AFL_TARGET_CRASH?=crashes
@ -103,10 +98,10 @@ clean:
cargo clean cargo clean
rm -rf bench/iai/target rm -rf bench/iai/target
git submodule deinit -f --all git submodule deinit -f --all
find tests -type l -path 'tests/suite/*.test' -print0 | xargs -0 rm -f find tests -type l -path 'tests/html-ut/ut/*.test' -print0 | xargs -0 rm -f
(cd tests/suite && make clean) (cd tests/html-ut && make clean)
rm -f tests/bench/*.dj rm -f tests/html-ref/*.dj
(cd tests/bench && make clean) (cd tests/html-ref && make clean)
find bench -type l -path 'bench/*.dj' -print0 | xargs -0 rm -f find bench -type l -path 'bench/*.dj' -print0 | xargs -0 rm -f
rm -rf tests/afl/out rm -rf tests/afl/out
(cd examples/jotdown_wasm && make clean) (cd examples/jotdown_wasm && make clean)

View file

@ -121,19 +121,19 @@ including:
- footnotes. - footnotes.
The HTML output is in some cases not exactly identical to the [reference The HTML output is in some cases not exactly identical to the [reference
implementation][djot-js]. There are two test suites that compares Jotdown with implementation][djot-js]. There are two test suites that compares Jotdown's
the reference implementation. One uses the unit tests of the reference HTML output with that of the reference implementation. One uses the unit tests
implementation and runs them with Jotdown. It can be run with: of the reference implementation and runs them with Jotdown. It can be run with:
``` ```
$ make suite $ make test_html_ut
``` ```
Another target uses the reference implementation to generate html output for Another target uses the reference implementation to generate html output for
its benchmark files and compares it to the output of Jotdown: its benchmark files and compares it to the output of Jotdown:
``` ```
$ make suite_bench $ make test_html_ref
``` ```
Note that it requires node in order to run the reference implementation. Note that it requires node in order to run the reference implementation.

12
tests/html-ref/Cargo.toml Normal file
View file

@ -0,0 +1,12 @@
[package]
name = "test-html-ref"
description = "Reference implementation HTML output comparison tests"
version = "0.1.0"
edition = "2021"
[dependencies]
jotdown = { path = "../.." }
[lib]
name = "test_html_ref"
path = "lib.rs"

View file

@ -6,8 +6,8 @@ TEST=${TEST_DJ:.dj=}
DJOT_JS=../../modules/djot.js DJOT_JS=../../modules/djot.js
DJOT_JS_SRC=$(shell find ${DJOT_JS}/src -name '.ts') DJOT_JS_SRC=$(shell find ${DJOT_JS}/src -name '.ts')
mod.rs: ${TEST_DJ} html ref.rs: ${TEST_DJ} html
echo "use crate::suite_test;" > $@ echo "use crate::compare;" > $@
for name in ${TEST}; do \ for name in ${TEST}; do \
name_snake=$$(basename -a $$name); \ name_snake=$$(basename -a $$name); \
skip_reason=$$(grep -E "^$${name_snake}:" skip | cut -d: -f2); \ skip_reason=$$(grep -E "^$${name_snake}:" skip | cut -d: -f2); \
@ -17,10 +17,8 @@ mod.rs: ${TEST_DJ} html
printf ' let src = r###"'; \ printf ' let src = r###"'; \
cat $$name.dj; \ cat $$name.dj; \
echo '"###;'; \ echo '"###;'; \
printf ' let expected = r###"'; \ printf ' let expected = "%s";' "$$name.html"; \
cat $$name.html; \ echo " compare!(src, expected);"; \
echo '"###;'; \
echo " suite_test!(src, expected);"; \
echo "}"; \ echo "}"; \
done >> $@ done >> $@
@ -36,6 +34,7 @@ djot-js: ${DJOT_JS_SRC}
chmod +x $@ chmod +x $@
clean: clean:
rm -f *.rs *.html rm -f ref.rs
rm -f *.html
rm -f html rm -f html
rm -f djot-js rm -f djot-js

17
tests/html-ref/build.rs Normal file
View file

@ -0,0 +1,17 @@
fn main() {
let has_dj = std::fs::read_dir(".").unwrap().any(|e| {
e.map_or(false, |e| {
e.path()
.extension()
.map_or(false, |ext| ext.to_str() == Some("dj"))
})
});
if has_dj {
let status = std::process::Command::new("make")
.status()
.expect("failed to execute make");
assert!(status.success());
} else {
std::fs::write("ref.rs", &[b'\n']).unwrap();
}
}

32
tests/html-ref/lib.rs Normal file
View file

@ -0,0 +1,32 @@
#[cfg(test)]
mod r#ref;
#[macro_export]
macro_rules! compare {
($src:expr, $expected:expr) => {
use jotdown::Render;
let src = $src;
let expected = std::fs::read_to_string($expected).expect("read failed");
let p = jotdown::Parser::new(src);
let mut actual = String::new();
jotdown::html::Renderer::default()
.push(p, &mut actual)
.unwrap();
assert_eq!(actual, expected, "\n{}", {
use std::io::Write;
let mut child = std::process::Command::new("diff")
.arg("--color=always")
.arg("-")
.arg($expected)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.spawn()
.expect("spawn diff failed");
let mut stdin = child.stdin.take().unwrap();
let actual = actual.clone();
std::thread::spawn(move || stdin.write_all(actual.as_bytes()).unwrap());
let stdout = child.wait_with_output().unwrap().stdout;
String::from_utf8(stdout).unwrap()
});
};
}

12
tests/html-ut/Cargo.toml Normal file
View file

@ -0,0 +1,12 @@
[package]
name = "test-html-ut"
description = "HTML output unit tests."
version = "0.1.0"
edition = "2021"
[dependencies]
jotdown = { path = "../.." }
[lib]
name = "test_html_ut"
path = "lib.rs"

View file

@ -2,25 +2,23 @@
.SUFFIXES: .test .rs .SUFFIXES: .test .rs
TEST=$(shell find . -name '*.test' | sort) TEST=$(shell find ut -name '*.test' | sort)
TEST_RS=${TEST:.test=.rs} TEST_RS=${TEST:.test=.rs}
BLACKLIST += djot_js_filters # lua filters not implemented BLACKLIST += djot_js_filters # lua filters not implemented
BLACKLIST += djot_js_symb # uses ast BLACKLIST += djot_js_symb # uses ast
BLACKLIST += djot_js_sourcepos # not parsable BLACKLIST += djot_js_sourcepos # not parsable
.PHONY: suite ut/mod.rs: ${TEST_RS}
suite: mod.rs mkdir -p ut
rm -f $@
mod.rs: ${TEST_RS}
printf "" > $@
for f in ${TEST}; do \ for f in ${TEST}; do \
name=$$(basename -s .test $$f); \ name=$$(basename -s .test $$f); \
echo ${BLACKLIST} | tr ' ' '\n' | grep -q $$name || echo "mod $$name;" >> $@; \ echo ${BLACKLIST} | tr ' ' '\n' | grep -q $$name || echo "mod $$name;" >> $@; \
done done
.test.rs: .test.rs:
gawk -fgen.awk $< > $@ gawk -fgen.awk $< | head -n-1 > $@
clean: clean:
rm -f *.rs rm -f ut/*.rs

6
tests/html-ut/build.rs Normal file
View file

@ -0,0 +1,6 @@
fn main() {
let status = std::process::Command::new("make")
.status()
.expect("failed to execute make");
assert!(status.success());
}

47
tests/html-ut/cmp.rs Normal file
View file

@ -0,0 +1,47 @@
#[macro_export]
macro_rules! compare {
($src:expr, $expected:expr) => {
use jotdown::Render;
let src = $src;
let expected = $expected;
let p = jotdown::Parser::new(src);
let mut actual = String::new();
jotdown::html::Renderer::default()
.push(p, &mut actual)
.unwrap();
assert_eq!(
actual.trim(),
expected.trim(),
concat!(
"\n",
"\x1b[0;1m========================= INPUT ============================\x1b[0m\n",
"\x1b[2m{}",
"\x1b[0;1m=================== ACTUAL vs EXPECTED =====================\x1b[0m\n",
"{}",
"\x1b[0;1m============================================================\x1b[0m\n",
),
$src,
{
let a = actual.trim().split('\n');
let b = expected.trim().split('\n');
let max = a.clone().count().max(b.clone().count());
let a_width = a.clone().map(|a| a.len()).max().unwrap_or(0);
a.chain(std::iter::repeat(""))
.zip(b.chain(std::iter::repeat("")))
.take(max)
.map(|(a, b)| {
format!(
"\x1b[{}m{:a_width$}\x1b[0m {}= \x1b[{}m{}\x1b[0m\n",
if a == b { "2" } else { "31" },
a,
if a == b { '=' } else { '!' },
if a == b { "2" } else { "32" },
b,
a_width = a_width,
)
})
.collect::<String>()
},
);
};
}

View file

@ -1,7 +1,7 @@
BEGIN { BEGIN {
FS=":" FS=":"
while (getline < "skip") skips[$1]=$2 while (getline < "skip") skips[$1]=$2
print "use crate::suite_test;" print "use crate::compare;"
print "" print ""
} }
@ -20,7 +20,7 @@ $0 ~ "^`{3,}$" {
close("src") close("src")
system("rm -f src") system("rm -f src")
print "\"##;" print "\"##;"
print " suite_test!(src, expected);" print " compare!(src, expected);"
print "}" print "}"
print "" print ""
} }

View file

@ -1,13 +1,8 @@
#[rustfmt::skip] #[cfg(test)]
#[cfg(feature = "suite_bench")] mod ut;
mod bench;
#[rustfmt::skip]
#[cfg(feature = "suite")]
mod suite;
#[cfg(any(feature = "suite", feature = "suite_bench"))]
#[macro_export] #[macro_export]
macro_rules! suite_test { macro_rules! compare {
($src:expr, $expected:expr) => { ($src:expr, $expected:expr) => {
use jotdown::Render; use jotdown::Render;
let src = $src; let src = $src;