fnmain/jotdown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

PR #11 criterion/iai benchmarks

Browse source 
closes #2
closes #11
This commit is contained in:
Noah Hellman 2023-02-14 22:08:14 +01:00
commit 1eb3bd12d0
13 changed files with 614 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
.github/workflows/ci.yml vendored
View file

@ -1,6 +1,9 @@
name: ci
on:
push:
branches:
- master
pull_request:
env:
@ -78,3 +81,27 @@ jobs:
run: |
echo core | sudo tee /proc/sys/kernel/core_pattern
make afl_quick
bench:
name: Benchmark
runs-on: ubuntu-latest
steps:
- name: "Checkout repo"
uses: actions/checkout@v3
- name: "Setup toolchain"
run: |
rustup update stable
rustup default stable
sudo apt-get install valgrind
- name: "Fetch benchmark inputs"
run: make bench
- name: "Benchmark"
run: |
cargo bench -p bench-crit all > crit
cargo bench -p bench-iai > iai
- name: "Artifact results"
uses: actions/upload-artifact@v3
with:
name: benchmark
path: |
crit
iai

349
Cargo.lock generated
View file

@ -2,18 +2,215 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "bench-crit"
version = "0.1.0"
dependencies = [
"bench-input",
"criterion",
"jotdown",
]
[[package]]
name = "bench-iai"
version = "0.1.0"
dependencies = [
"bench-input",
"iai",
"jotdown",
]
[[package]]
name = "bench-input"
version = "0.1.0"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bumpalo"
version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "ciborium"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
[[package]]
name = "ciborium-ll"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "clap"
version = "3.2.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
dependencies = [
"bitflags",
"clap_lex",
"indexmap",
"textwrap",
]
[[package]]
name = "clap_lex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
dependencies = [
"os_str_bytes",
]
[[package]]
name = "criterion"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
dependencies = [
"anes",
"atty",
"cast",
"ciborium",
"clap",
"criterion-plot",
"itertools",
"lazy_static",
"num-traits",
"oorandom",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "either"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "half"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]]
name = "iai"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71a816c97c42258aa5834d07590b718b4c9a598944cd39a52dc25b351185d678"
[[package]]
name = "indexmap"
version = "1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
dependencies = [
"autocfg",
"hashbrown",
]
[[package]]
name = "itertools"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
[[package]]
name = "jotdown"
version = "0.1.0"
@ -36,6 +233,18 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.139"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
[[package]]
name = "log"
version = "0.4.17"
@ -45,12 +254,33 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
[[package]]
name = "oorandom"
version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "os_str_bytes"
version = "6.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
[[package]]
name = "proc-macro2"
version = "1.0.51"
@ -69,6 +299,67 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "regex"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
dependencies = [
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
[[package]]
name = "ryu"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "serde"
version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "syn"
version = "1.0.107"
@ -80,12 +371,39 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "textwrap"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "unicode-ident"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
[[package]]
name = "walkdir"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
dependencies = [
"same-file",
"winapi",
"winapi-util",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.84"
@ -149,3 +467,34 @@ dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

4
Cargo.toml
View file

@ -13,13 +13,15 @@ documentation = "https://docs.rs/jotdown"
exclude = [
".gitmodules",
"Makefile",
"benches",
"modules",
"tests",
]
[workspace]
members = [
"bench/criterion",
"bench/iai",
"bench/input",
"examples/jotdown_wasm",
]
exclude = [

4
Makefile
View file

@ -47,7 +47,7 @@ suite_bench:
bench:
git submodule update --init modules/djot.js
for f in $$(find modules/djot.js/bench -name '*.dj' | xargs basename -a); do \
ln -fs ../modules/djot.js/bench/$$f benches/$$f; \
ln -fs ../modules/djot.js/bench/$$f bench/$$f; \
done
cov: suite suite_bench
@ -96,6 +96,6 @@ clean:
(cd tests/suite && make clean)
rm -f tests/bench/*.dj
(cd tests/bench && make clean)
rm -f benches/*.dj
rm -f bench/*.dj
rm -rf tests/afl/out
(cd examples/jotdown_wasm && make clean)

33
README.md
View file

@ -78,6 +78,8 @@ WebAssembly backend.
## Status
### Correctness
As of writing, Jotdown implements all the current features of the Djot syntax,
including:
@ -125,3 +127,34 @@ $ make suite_bench
```
Note that it requires node in order to run the reference implementation.
### Performance
There are benchmarks available to measure the performance. The input files are
borrowed from the [reference implementation][djot-js]. To fetch the input files
symlink them to the bench directory, run:
```
make bench
```
There are two separate benchmarks suites that can be run; criterion and iai.
Criterion measures statistical real-time performance while iai measures exact
number of executed instructions and memory accesses. To run the criterion
benchmarks, use
```
cargo bench -p bench-crit [filter]
```
Alternatively, if `cargo-criterion` is installed:
```
cargo criterion -p bench-crit -- [filter]
```
To run the iai benchmarks, use
```
cargo bench -p bench-iai
```

14
bench/criterion/Cargo.toml Normal file
View file

@ -0,0 +1,14 @@
[package]
name = "bench-crit"
version = "0.1.0"
edition = "2021"
[dev-dependencies]
criterion = { version = "0.4", default-features = false }
bench-input = { path = "../input" }
jotdown = { path = "../../" }
[[bench]]
name = "criterion"
path = "main.rs"
harness = false

86
bench/criterion/main.rs Normal file
View file

@ -0,0 +1,86 @@
use criterion::criterion_group;
use criterion::criterion_main;
use jotdown::Render;
fn gen_block(c: &mut criterion::Criterion) {
let mut group = c.benchmark_group("block");
for (name, input) in bench_input::INPUTS {
group.throughput(criterion::Throughput::Bytes(input.len() as u64));
group.bench_with_input(
criterion::BenchmarkId::from_parameter(name),
input,
|b, &input| {
b.iter(|| jotdown::Parser::new(input));
},
);
}
}
criterion_group!(block, gen_block);
fn gen_inline(c: &mut criterion::Criterion) {
let mut group = c.benchmark_group("inline");
for (name, input) in bench_input::INPUTS {
group.throughput(criterion::Throughput::Bytes(input.len() as u64));
group.bench_with_input(
criterion::BenchmarkId::from_parameter(name),
input,
|b, &input| {
b.iter_batched(
|| jotdown::Parser::new(input),
|p| p.last().unwrap(),
criterion::BatchSize::SmallInput,
);
},
);
}
}
criterion_group!(inline, gen_inline);
fn gen_html(c: &mut criterion::Criterion) {
let mut group = c.benchmark_group("html");
for (name, input) in bench_input::INPUTS {
group.throughput(criterion::Throughput::Elements(
jotdown::Parser::new(input).count() as u64,
));
group.bench_with_input(
criterion::BenchmarkId::from_parameter(name),
input,
|b, &input| {
b.iter_batched(
|| jotdown::Parser::new(input).collect::<Vec<_>>(),
|p| {
let mut s = String::new();
jotdown::html::Renderer.push(p.into_iter(), &mut s).unwrap();
s
},
criterion::BatchSize::SmallInput,
);
},
);
}
}
criterion_group!(html, gen_html);
fn gen_full(c: &mut criterion::Criterion) {
let mut group = c.benchmark_group("full");
for (name, input) in bench_input::INPUTS {
group.throughput(criterion::Throughput::Bytes(input.len() as u64));
group.bench_with_input(
criterion::BenchmarkId::from_parameter(name),
input,
|b, &input| {
b.iter_with_large_drop(|| {
let mut s = String::new();
jotdown::html::Renderer
.push(jotdown::Parser::new(input), &mut s)
.unwrap();
s
});
},
);
}
}
criterion_group!(full, gen_full);
criterion_main!(block, inline, html, full);

14
bench/iai/Cargo.toml Normal file
View file

@ -0,0 +1,14 @@
[package]
name = "bench-iai"
version = "0.1.0"
edition = "2021"
[dev-dependencies]
iai = { version = "0.1", default-features = false }
bench-input = { path = "../input" }
jotdown = { path = "../../", features = ["html", "deterministic"] }
[[bench]]
name = "iai"
path = "main.rs"
harness = false

21
bench/iai/main.rs Normal file
View file

@ -0,0 +1,21 @@
use iai::black_box;
use jotdown::Render;
fn block() -> jotdown::Parser<'static> {
jotdown::Parser::new(black_box(bench_input::ALL))
}
fn block_inline() -> Option<jotdown::Event<'static>> {
black_box(jotdown::Parser::new(black_box(bench_input::ALL))).last()
}
fn full() -> String {
let mut s = String::new();
jotdown::html::Renderer
.push(jotdown::Parser::new(bench_input::ALL), &mut s)
.unwrap();
s
}
iai::main!(block, block_inline, full);

7
bench/input/Cargo.toml Normal file
View file

@ -0,0 +1,7 @@
[package]
name = "bench-input"
version = "0.1.0"
edition = "2021"
[lib]
path = "lib.rs"

57
bench/input/build.rs Normal file
View file

@ -0,0 +1,57 @@
use std::io::Write;
fn main() {
let inputs = std::fs::read_dir("..")
.unwrap()
.filter_map(|entry| {
let entry = entry.ok()?;
if let Some(name) = entry.file_name().to_str() {
if let Some(name) = name.strip_suffix(".dj") {
if entry.file_type().map_or(false, |ty| !ty.is_dir()) {
let name_snake = name.replace('-', "_");
let input = std::fs::read_to_string(
std::path::Path::new("..").join(entry.file_name()),
)
.ok()?;
return Some((name_snake, input));
}
}
}
None
})
.collect::<Vec<_>>();
let out_dir = std::env::var_os("OUT_DIR").unwrap();
let mut out = std::fs::File::create(std::path::Path::new(&out_dir).join("lib.rs")).unwrap();
inputs.iter().for_each(|(name, input)| {
write!(
out,
"#[allow(dead_code)]\nconst {}: &str = r###\"{}\"###;",
name.to_uppercase(),
input,
)
.unwrap()
});
write!(
out,
"#[allow(dead_code)]\npub const ALL: &str = r###\"{}\"###;",
inputs.iter().map(|(_, s)| s.as_str()).collect::<String>(),
)
.unwrap();
write!(
out,
"#[allow(dead_code)]\npub const INPUTS: &[(&str, &str)] = &[{}];",
inputs
.iter()
.map(|(n, _)| n.as_ref())
.chain(std::iter::once("all"))
.map(|n| format!("(\"{}\", {}),", n, n.to_uppercase()))
.collect::<String>(),
)
.unwrap();
println!("cargo:rerun-if-change=always_rerun");
}

1
bench/input/lib.rs Normal file
View file

@ -0,0 +1 @@
include!(concat!(env!("OUT_DIR"), "/lib.rs"));

19
benches/bench.sh
View file

@ -1,19 +0,0 @@
#!/bin/sh
cmd=$*
test_html=$(echo "abc" | $cmd | tr -d ' \n')
[ "$test_html" != "<p>abc</p>" ] && echo "failed simple gen: '$test_html'" && exit 1
for f in *.dj; do
tmp=$(mktemp)
for i in $(seq 500); do
cat "$f" >> $tmp
done
t=$(cat "$tmp" \
| (LANG=C time -p $cmd > /dev/null) 2>&1 \
| grep real \
| awk '{print $2}')
printf "$t\t$f\n"
rm -f "$tmp"
done | sort -rn