PR #11 criterion/iai benchmarks

closes #2
closes #11

.github/workflows/ci.yml

@@ -1,6 +1,9 @@
 name: ci
 
 on:
+  push:
+    branches:
+      - master
   pull_request:
 
 env:
@@ -78,3 +81,27 @@ jobs:
         run: |
           echo core | sudo tee /proc/sys/kernel/core_pattern
           make afl_quick
+  bench:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout repo"
+        uses: actions/checkout@v3
+      - name: "Setup toolchain"
+        run: |
+          rustup update stable
+          rustup default stable
+          sudo apt-get install valgrind
+      - name: "Fetch benchmark inputs"
+        run: make bench
+      - name: "Benchmark"
+        run: |
+          cargo bench -p bench-crit all > crit
+          cargo bench -p bench-iai > iai
+      - name: "Artifact results"
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmark
+          path: |
+            crit
+            iai

Cargo.lock

@@ -2,18 +2,215 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bench-crit"
+version = "0.1.0"
+dependencies = [
+ "bench-input",
+ "criterion",
+ "jotdown",
+]
+
+[[package]]
+name = "bench-iai"
+version = "0.1.0"
+dependencies = [
+ "bench-input",
+ "iai",
+ "jotdown",
+]
+
+[[package]]
+name = "bench-input"
+version = "0.1.0"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
 [[package]]
 name = "bumpalo"
 version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "ciborium"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
+[[package]]
+name = "clap"
+version = "3.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
+dependencies = [
+ "bitflags",
+ "clap_lex",
+ "indexmap",
+ "textwrap",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+dependencies = [
+ "os_str_bytes",
+]
+
+[[package]]
+name = "criterion"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
+dependencies = [
+ "anes",
+ "atty",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "either"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+
+[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "iai"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71a816c97c42258aa5834d07590b718b4c9a598944cd39a52dc25b351185d678"
+
+[[package]]
+name = "indexmap"
+version = "1.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
+
 [[package]]
 name = "jotdown"
 version = "0.1.0"
@@ -36,6 +233,18 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.139"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+
 [[package]]
 name = "log"
 version = "0.4.17"
@@ -45,12 +254,33 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
 
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
+[[package]]
+name = "os_str_bytes"
+version = "6.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.51"
@@ -69,6 +299,67 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "regex"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
+dependencies = [
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
+
+[[package]]
+name = "ryu"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.152"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.152"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.107"
@@ -80,12 +371,39 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
+
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
+[[package]]
+name = "walkdir"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
+dependencies = [
+ "same-file",
+ "winapi",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.84"
@@ -149,3 +467,34 @@ dependencies = [
  "js-sys",
  "wasm-bindgen",
 ]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

Cargo.toml

@@ -13,13 +13,15 @@ documentation = "https://docs.rs/jotdown"
 exclude = [
     ".gitmodules",
     "Makefile",
-    "benches",
     "modules",
     "tests",
 ]
 
 [workspace]
 members = [
+    "bench/criterion",
+    "bench/iai",
+    "bench/input",
     "examples/jotdown_wasm",
 ]
 exclude = [

Makefile

@@ -47,7 +47,7 @@ suite_bench:
 bench:
 	git submodule update --init modules/djot.js
 	for f in $$(find modules/djot.js/bench -name '*.dj' | xargs basename -a); do \
-		ln -fs ../modules/djot.js/bench/$$f benches/$$f; \
+		ln -fs ../modules/djot.js/bench/$$f bench/$$f; \
 	done
 
 cov: suite suite_bench
@@ -96,6 +96,6 @@ clean:
 	(cd tests/suite && make clean)
 	rm -f tests/bench/*.dj
 	(cd tests/bench && make clean)
-	rm -f benches/*.dj
+	rm -f bench/*.dj
 	rm -rf tests/afl/out
 	(cd examples/jotdown_wasm && make clean)

README.md

@@ -78,6 +78,8 @@ WebAssembly backend.
 
 ## Status
 
+### Correctness
+
 As of writing, Jotdown implements all the current features of the Djot syntax,
 including:
 
@@ -125,3 +127,34 @@ $ make suite_bench
 ```
 
 Note that it requires node in order to run the reference implementation.
+
+### Performance
+
+There are benchmarks available to measure the performance. The input files are
+borrowed from the [reference implementation][djot-js]. To fetch the input files
+symlink them to the bench directory, run:
+
+```
+make bench
+```
+
+There are two separate benchmarks suites that can be run; criterion and iai.
+Criterion measures statistical real-time performance while iai measures exact
+number of executed instructions and memory accesses. To run the criterion
+benchmarks, use
+
+```
+cargo bench -p bench-crit [filter]
+```
+
+Alternatively, if `cargo-criterion` is installed:
+
+```
+cargo criterion -p bench-crit -- [filter]
+```
+
+To run the iai benchmarks, use
+
+```
+cargo bench -p bench-iai
+```

bench/criterion/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "bench-crit"
+version = "0.1.0"
+edition = "2021"
+
+[dev-dependencies]
+criterion = { version = "0.4", default-features = false }
+bench-input = { path = "../input" }
+jotdown = { path = "../../" }
+
+[[bench]]
+name = "criterion"
+path = "main.rs"
+harness = false

bench/criterion/main.rs

@@ -0,0 +1,86 @@
+use criterion::criterion_group;
+use criterion::criterion_main;
+
+use jotdown::Render;
+
+fn gen_block(c: &mut criterion::Criterion) {
+    let mut group = c.benchmark_group("block");
+    for (name, input) in bench_input::INPUTS {
+        group.throughput(criterion::Throughput::Bytes(input.len() as u64));
+        group.bench_with_input(
+            criterion::BenchmarkId::from_parameter(name),
+            input,
+            |b, &input| {
+                b.iter(|| jotdown::Parser::new(input));
+            },
+        );
+    }
+}
+criterion_group!(block, gen_block);
+
+fn gen_inline(c: &mut criterion::Criterion) {
+    let mut group = c.benchmark_group("inline");
+    for (name, input) in bench_input::INPUTS {
+        group.throughput(criterion::Throughput::Bytes(input.len() as u64));
+        group.bench_with_input(
+            criterion::BenchmarkId::from_parameter(name),
+            input,
+            |b, &input| {
+                b.iter_batched(
+                    || jotdown::Parser::new(input),
+                    |p| p.last().unwrap(),
+                    criterion::BatchSize::SmallInput,
+                );
+            },
+        );
+    }
+}
+criterion_group!(inline, gen_inline);
+
+fn gen_html(c: &mut criterion::Criterion) {
+    let mut group = c.benchmark_group("html");
+    for (name, input) in bench_input::INPUTS {
+        group.throughput(criterion::Throughput::Elements(
+            jotdown::Parser::new(input).count() as u64,
+        ));
+        group.bench_with_input(
+            criterion::BenchmarkId::from_parameter(name),
+            input,
+            |b, &input| {
+                b.iter_batched(
+                    || jotdown::Parser::new(input).collect::<Vec<_>>(),
+                    |p| {
+                        let mut s = String::new();
+                        jotdown::html::Renderer.push(p.into_iter(), &mut s).unwrap();
+                        s
+                    },
+                    criterion::BatchSize::SmallInput,
+                );
+            },
+        );
+    }
+}
+criterion_group!(html, gen_html);
+
+fn gen_full(c: &mut criterion::Criterion) {
+    let mut group = c.benchmark_group("full");
+    for (name, input) in bench_input::INPUTS {
+        group.throughput(criterion::Throughput::Bytes(input.len() as u64));
+        group.bench_with_input(
+            criterion::BenchmarkId::from_parameter(name),
+            input,
+            |b, &input| {
+                b.iter_with_large_drop(|| {
+                    let mut s = String::new();
+                    jotdown::html::Renderer
+                        .push(jotdown::Parser::new(input), &mut s)
+                        .unwrap();
+                    s
+                });
+            },
+        );
+    }
+}
+criterion_group!(full, gen_full);
+
+criterion_main!(block, inline, html, full);

bench/iai/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "bench-iai"
+version = "0.1.0"
+edition = "2021"
+
+[dev-dependencies]
+iai = { version = "0.1", default-features = false }
+bench-input = { path = "../input" }
+jotdown = { path = "../../", features = ["html", "deterministic"] }
+
+[[bench]]
+name = "iai"
+path = "main.rs"
+harness = false

bench/iai/main.rs

@@ -0,0 +1,21 @@
+use iai::black_box;
+
+use jotdown::Render;
+
+fn block() -> jotdown::Parser<'static> {
+    jotdown::Parser::new(black_box(bench_input::ALL))
+}
+
+fn block_inline() -> Option<jotdown::Event<'static>> {
+    black_box(jotdown::Parser::new(black_box(bench_input::ALL))).last()
+}
+
+fn full() -> String {
+    let mut s = String::new();
+    jotdown::html::Renderer
+        .push(jotdown::Parser::new(bench_input::ALL), &mut s)
+        .unwrap();
+    s
+}
+
+iai::main!(block, block_inline, full);

bench/input/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "bench-input"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+path = "lib.rs"

bench/input/build.rs

@@ -0,0 +1,57 @@
+use std::io::Write;
+
+fn main() {
+    let inputs = std::fs::read_dir("..")
+        .unwrap()
+        .filter_map(|entry| {
+            let entry = entry.ok()?;
+            if let Some(name) = entry.file_name().to_str() {
+                if let Some(name) = name.strip_suffix(".dj") {
+                    if entry.file_type().map_or(false, |ty| !ty.is_dir()) {
+                        let name_snake = name.replace('-', "_");
+                        let input = std::fs::read_to_string(
+                            std::path::Path::new("..").join(entry.file_name()),
+                        )
+                        .ok()?;
+                        return Some((name_snake, input));
+                    }
+                }
+            }
+            None
+        })
+        .collect::<Vec<_>>();
+
+    let out_dir = std::env::var_os("OUT_DIR").unwrap();
+    let mut out = std::fs::File::create(std::path::Path::new(&out_dir).join("lib.rs")).unwrap();
+
+    inputs.iter().for_each(|(name, input)| {
+        write!(
+            out,
+            "#[allow(dead_code)]\nconst {}: &str = r###\"{}\"###;",
+            name.to_uppercase(),
+            input,
+        )
+        .unwrap()
+    });
+
+    write!(
+        out,
+        "#[allow(dead_code)]\npub const ALL: &str = r###\"{}\"###;",
+        inputs.iter().map(|(_, s)| s.as_str()).collect::<String>(),
+    )
+    .unwrap();
+
+    write!(
+        out,
+        "#[allow(dead_code)]\npub const INPUTS: &[(&str, &str)] = &[{}];",
+        inputs
+            .iter()
+            .map(|(n, _)| n.as_ref())
+            .chain(std::iter::once("all"))
+            .map(|n| format!("(\"{}\", {}),", n, n.to_uppercase()))
+            .collect::<String>(),
+    )
+    .unwrap();
+
+    println!("cargo:rerun-if-change=always_rerun");
+}

bench/input/lib.rs

@@ -0,0 +1 @@
+include!(concat!(env!("OUT_DIR"), "/lib.rs"));

benches/bench.sh

@@ -1,19 +0,0 @@
-#!/bin/sh
-
-cmd=$*
-
-test_html=$(echo "abc" | $cmd | tr -d ' \n')
-[ "$test_html" != "<p>abc</p>" ] && echo "failed simple gen: '$test_html'" && exit 1
-
-for f in *.dj; do
-    tmp=$(mktemp)
-    for i in $(seq 500); do
-        cat "$f" >> $tmp
-    done
-    t=$(cat "$tmp" \
-        | (LANG=C time -p $cmd > /dev/null) 2>&1 \
-        | grep real \
-        | awk '{print $2}')
-    printf "$t\t$f\n"
-    rm -f "$tmp"
-done | sort -rn