mv suite{_bench} to test-html-{ut,ref} crates

- allow compiling/running html tests without compiling main crate tests
  (useful when e.g. making type changes to events but html unaffected)
- avoid need for future flags in main crate
This commit is contained in:
Noah Hellman 2023-04-28 20:58:08 +02:00
parent d2a46663f1
commit 3cea79a122
18 changed files with 182 additions and 55 deletions

View file

@ -34,8 +34,8 @@ jobs:
RUSTDOCFLAGS: -D warnings
run: |
make check
suite:
name: Build and run external tests
test_html:
name: Build and run HTML tests
runs-on: ubuntu-latest
steps:
- name: "Checkout repo"
@ -44,14 +44,14 @@ jobs:
run: |
rustup update 1.56
rustup default 1.56
- name: "Run unit tests"
run: make suite
- name: "Run HTML unit tests"
run: make test_html_ut
- name: "Setup node"
uses: actions/setup-node@v3
with:
node-version: 18
- name: "Compare benchmark files"
run: make suite_bench
- name: "Compare HTML with reference implementation"
run: make test_html_ref
lint:
name: Lint
runs-on: ubuntu-latest

14
Cargo.lock generated
View file

@ -458,6 +458,20 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "test-html-ref"
version = "0.1.0"
dependencies = [
"jotdown",
]
[[package]]
name = "test-html-ut"
version = "0.1.0"
dependencies = [
"jotdown",
]
[[package]]
name = "textwrap"
version = "0.16.0"

View file

@ -26,6 +26,8 @@ members = [
"bench/iai",
"bench/input",
"examples/jotdown_wasm",
"tests/html-ref",
"tests/html-ut",
]
exclude = [
"tests/afl",
@ -39,6 +41,4 @@ doc = false
[features]
default = ["html"]
html = [] # html renderer and minimal cli binary
suite = [] # test suite
suite_bench = [] # bench test suite
deterministic = [] # for stable fuzzing

View file

@ -25,24 +25,22 @@ check:
cargo test --workspace
cargo test --workspace --no-default-features
.PHONY: suite
suite:
.PHONY: test_html_ut
test_html_ut:
git submodule update --init modules/djot.js
for f in $$(find modules/djot.js/test -name '*.test' | xargs basename -a); do \
ln -fs ../../modules/djot.js/test/$$f tests/suite/djot_js_$$f; \
ln -fs ../../../modules/djot.js/test/$$f tests/html-ut/ut/djot_js_$$f; \
done
(cd tests/suite && make)
cargo test --features suite suite::
cargo test -p test-html-ut
.PHONY: suite_bench
suite_bench:
.PHONY: test_html_ref
test_html_ref:
git submodule update --init modules/djot.js
for f in $$(find modules/djot.js/bench -name '*.dj' | xargs basename -a); do \
dst=$$(echo $$f | sed 's/-/_/g'); \
ln -fs ../../modules/djot.js/bench/$$f tests/bench/$$dst; \
ln -fs ../../modules/djot.js/bench/$$f tests/html-ref/$$dst; \
done
(cd tests/bench && make)
cargo test --features suite_bench bench::
cargo test -p test-html-ref
.PHONY: bench
bench:
@ -52,9 +50,6 @@ bench:
ln -fs ../../modules/djot.js/bench/$$f bench/input/$$dst; \
done
cov: suite suite_bench
LLVM_COV=llvm-cov LLVM_PROFDATA=llvm-profdata cargo llvm-cov --features=suite,suite_bench --workspace --html --ignore-run-fail
AFL_TARGET?=parse
AFL_JOBS?=1
AFL_TARGET_CRASH?=crashes
@ -103,10 +98,10 @@ clean:
cargo clean
rm -rf bench/iai/target
git submodule deinit -f --all
find tests -type l -path 'tests/suite/*.test' -print0 | xargs -0 rm -f
(cd tests/suite && make clean)
rm -f tests/bench/*.dj
(cd tests/bench && make clean)
find tests -type l -path 'tests/html-ut/ut/*.test' -print0 | xargs -0 rm -f
(cd tests/html-ut && make clean)
rm -f tests/html-ref/*.dj
(cd tests/html-ref && make clean)
find bench -type l -path 'bench/*.dj' -print0 | xargs -0 rm -f
rm -rf tests/afl/out
(cd examples/jotdown_wasm && make clean)

View file

@ -121,19 +121,19 @@ including:
- footnotes.
The HTML output is in some cases not exactly identical to the [reference
implementation][djot-js]. There are two test suites that compares Jotdown with
the reference implementation. One uses the unit tests of the reference
implementation and runs them with Jotdown. It can be run with:
implementation][djot-js]. There are two test suites that compares Jotdown's
HTML output with that of the reference implementation. One uses the unit tests
of the reference implementation and runs them with Jotdown. It can be run with:
```
$ make suite
$ make test_html_ut
```
Another target uses the reference implementation to generate html output for
its benchmark files and compares it to the output of Jotdown:
```
$ make suite_bench
$ make test_html_ref
```
Note that it requires node in order to run the reference implementation.

12
tests/html-ref/Cargo.toml Normal file
View file

@ -0,0 +1,12 @@
[package]
name = "test-html-ref"
description = "Reference implementation HTML output comparison tests"
version = "0.1.0"
edition = "2021"
[dependencies]
jotdown = { path = "../.." }
[lib]
name = "test_html_ref"
path = "lib.rs"

View file

@ -6,8 +6,8 @@ TEST=${TEST_DJ:.dj=}
DJOT_JS=../../modules/djot.js
DJOT_JS_SRC=$(shell find ${DJOT_JS}/src -name '.ts')
mod.rs: ${TEST_DJ} html
echo "use crate::suite_test;" > $@
ref.rs: ${TEST_DJ} html
echo "use crate::compare;" > $@
for name in ${TEST}; do \
name_snake=$$(basename -a $$name); \
skip_reason=$$(grep -E "^$${name_snake}:" skip | cut -d: -f2); \
@ -17,10 +17,8 @@ mod.rs: ${TEST_DJ} html
printf ' let src = r###"'; \
cat $$name.dj; \
echo '"###;'; \
printf ' let expected = r###"'; \
cat $$name.html; \
echo '"###;'; \
echo " suite_test!(src, expected);"; \
printf ' let expected = "%s";' "$$name.html"; \
echo " compare!(src, expected);"; \
echo "}"; \
done >> $@
@ -36,6 +34,7 @@ djot-js: ${DJOT_JS_SRC}
chmod +x $@
clean:
rm -f *.rs *.html
rm -f ref.rs
rm -f *.html
rm -f html
rm -f djot-js

17
tests/html-ref/build.rs Normal file
View file

@ -0,0 +1,17 @@
fn main() {
let has_dj = std::fs::read_dir(".").unwrap().any(|e| {
e.map_or(false, |e| {
e.path()
.extension()
.map_or(false, |ext| ext.to_str() == Some("dj"))
})
});
if has_dj {
let status = std::process::Command::new("make")
.status()
.expect("failed to execute make");
assert!(status.success());
} else {
std::fs::write("ref.rs", &[b'\n']).unwrap();
}
}

32
tests/html-ref/lib.rs Normal file
View file

@ -0,0 +1,32 @@
#[cfg(test)]
mod r#ref;
#[macro_export]
macro_rules! compare {
($src:expr, $expected:expr) => {
use jotdown::Render;
let src = $src;
let expected = std::fs::read_to_string($expected).expect("read failed");
let p = jotdown::Parser::new(src);
let mut actual = String::new();
jotdown::html::Renderer::default()
.push(p, &mut actual)
.unwrap();
assert_eq!(actual, expected, "\n{}", {
use std::io::Write;
let mut child = std::process::Command::new("diff")
.arg("--color=always")
.arg("-")
.arg($expected)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.spawn()
.expect("spawn diff failed");
let mut stdin = child.stdin.take().unwrap();
let actual = actual.clone();
std::thread::spawn(move || stdin.write_all(actual.as_bytes()).unwrap());
let stdout = child.wait_with_output().unwrap().stdout;
String::from_utf8(stdout).unwrap()
});
};
}

12
tests/html-ut/Cargo.toml Normal file
View file

@ -0,0 +1,12 @@
[package]
name = "test-html-ut"
description = "HTML output unit tests."
version = "0.1.0"
edition = "2021"
[dependencies]
jotdown = { path = "../.." }
[lib]
name = "test_html_ut"
path = "lib.rs"

View file

@ -2,25 +2,23 @@
.SUFFIXES: .test .rs
TEST=$(shell find . -name '*.test' | sort)
TEST=$(shell find ut -name '*.test' | sort)
TEST_RS=${TEST:.test=.rs}
BLACKLIST += djot_js_filters # lua filters not implemented
BLACKLIST += djot_js_symb # uses ast
BLACKLIST += djot_js_sourcepos # not parsable
.PHONY: suite
suite: mod.rs
mod.rs: ${TEST_RS}
printf "" > $@
ut/mod.rs: ${TEST_RS}
mkdir -p ut
rm -f $@
for f in ${TEST}; do \
name=$$(basename -s .test $$f); \
echo ${BLACKLIST} | tr ' ' '\n' | grep -q $$name || echo "mod $$name;" >> $@; \
done
.test.rs:
gawk -fgen.awk $< > $@
gawk -fgen.awk $< | head -n-1 > $@
clean:
rm -f *.rs
rm -f ut/*.rs

6
tests/html-ut/build.rs Normal file
View file

@ -0,0 +1,6 @@
fn main() {
let status = std::process::Command::new("make")
.status()
.expect("failed to execute make");
assert!(status.success());
}

47
tests/html-ut/cmp.rs Normal file
View file

@ -0,0 +1,47 @@
#[macro_export]
macro_rules! compare {
($src:expr, $expected:expr) => {
use jotdown::Render;
let src = $src;
let expected = $expected;
let p = jotdown::Parser::new(src);
let mut actual = String::new();
jotdown::html::Renderer::default()
.push(p, &mut actual)
.unwrap();
assert_eq!(
actual.trim(),
expected.trim(),
concat!(
"\n",
"\x1b[0;1m========================= INPUT ============================\x1b[0m\n",
"\x1b[2m{}",
"\x1b[0;1m=================== ACTUAL vs EXPECTED =====================\x1b[0m\n",
"{}",
"\x1b[0;1m============================================================\x1b[0m\n",
),
$src,
{
let a = actual.trim().split('\n');
let b = expected.trim().split('\n');
let max = a.clone().count().max(b.clone().count());
let a_width = a.clone().map(|a| a.len()).max().unwrap_or(0);
a.chain(std::iter::repeat(""))
.zip(b.chain(std::iter::repeat("")))
.take(max)
.map(|(a, b)| {
format!(
"\x1b[{}m{:a_width$}\x1b[0m {}= \x1b[{}m{}\x1b[0m\n",
if a == b { "2" } else { "31" },
a,
if a == b { '=' } else { '!' },
if a == b { "2" } else { "32" },
b,
a_width = a_width,
)
})
.collect::<String>()
},
);
};
}

View file

@ -1,7 +1,7 @@
BEGIN {
FS=":"
while (getline < "skip") skips[$1]=$2
print "use crate::suite_test;"
print "use crate::compare;"
print ""
}
@ -20,7 +20,7 @@ $0 ~ "^`{3,}$" {
close("src")
system("rm -f src")
print "\"##;"
print " suite_test!(src, expected);"
print " compare!(src, expected);"
print "}"
print ""
}

View file

@ -1,13 +1,8 @@
#[rustfmt::skip]
#[cfg(feature = "suite_bench")]
mod bench;
#[rustfmt::skip]
#[cfg(feature = "suite")]
mod suite;
#[cfg(test)]
mod ut;
#[cfg(any(feature = "suite", feature = "suite_bench"))]
#[macro_export]
macro_rules! suite_test {
macro_rules! compare {
($src:expr, $expected:expr) => {
use jotdown::Render;
let src = $src;