PR #13 Setup CI

closes #9
closes #13
This commit is contained in:
Noah Hellman 2023-02-14 16:15:48 +01:00
commit 0fa0f8fd5b
14 changed files with 194 additions and 267 deletions

80
.github/workflows/ci.yml vendored Normal file
View file

@ -0,0 +1,80 @@
name: ci
on:
pull_request:
env:
CARGO_TERM_COLOR: always
jobs:
test:
name: Build and run tests
runs-on: ubuntu-latest
strategy:
matrix:
toolchain:
- "1.56"
- stable
steps:
- name: "Checkout repo"
uses: actions/checkout@v3
- name: "Setup toolchain"
run: |
rustup update ${{ matrix.toolchain }}
rustup default ${{ matrix.toolchain }}
- name: "Build"
run: |
make all
cargo build --workspace --no-default-features
- name: "Run tests"
env:
RUSTDOCFLAGS: -D warnings
run: |
make check
suite:
name: Build and run external tests
runs-on: ubuntu-latest
steps:
- name: "Checkout repo"
uses: actions/checkout@v3
- name: "Setup toolchain"
run: |
rustup update 1.56
rustup default 1.56
- name: "Run unit tests"
run: make suite
- name: "Setup node"
uses: actions/setup-node@v3
with:
node-version: 18
- name: "Compare benchmark files"
run: make suite_bench
lint:
name: Lint
runs-on: ubuntu-latest
steps:
- name: "Checkout"
uses: actions/checkout@v3
- name: "Setup toolchain"
run: |
rustup update 1.56
rustup default 1.56
rustup component add rustfmt
rustup component add clippy
- name: "Check linting"
run: make lint
fuzz:
name: Fuzz
runs-on: ubuntu-latest
steps:
- name: "Checkout"
uses: actions/checkout@v3
- name: "Setup toolchain"
run: |
rustup update nightly
rustup default nightly
cargo install afl
- name: "Fuzz"
run: |
echo core | sudo tee /proc/sys/kernel/core_pattern
make afl_quick

2
.gitmodules vendored
View file

@ -1,3 +1,3 @@
[submodule "jgm/djot.js"] [submodule "jgm/djot.js"]
path = modules/djot.js path = modules/djot.js
url = git@github.com:jgm/djot.js.git url = https://github.com/jgm/djot.js.git

233
Cargo.lock generated
View file

@ -2,130 +2,22 @@
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3 version = 3
[[package]]
name = "afl"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c0f4180b6d7095a1c3130f1aadead4ece7fb3094dc724c701adb30a92a95228"
dependencies = [
"cc",
"clap",
"libc",
"rustc_version",
"xdg",
]
[[package]]
name = "ansi_term"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
dependencies = [
"winapi",
]
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.12.0" version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
[[package]]
name = "cc"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "1.0.0" version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clap"
version = "2.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
"ansi_term",
"atty",
"bitflags",
"strsim",
"textwrap",
"unicode-width",
"vec_map",
]
[[package]]
name = "dirs"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "getrandom"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "jotdown" name = "jotdown"
version = "0.1.0" version = "0.1.0"
[[package]]
name = "jotdown-afl"
version = "0.1.0"
dependencies = [
"afl",
"jotdown",
]
[[package]] [[package]]
name = "jotdown_wasm" name = "jotdown_wasm"
version = "0.1.0" version = "0.1.0"
@ -144,12 +36,6 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "libc"
version = "0.2.139"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.17" version = "0.4.17"
@ -183,47 +69,6 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom",
"redox_syscall",
"thiserror",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver",
]
[[package]]
name = "semver"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
[[package]]
name = "strsim"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]] [[package]]
name = "syn" name = "syn"
version = "1.0.107" version = "1.0.107"
@ -235,59 +80,12 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "textwrap"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
dependencies = [
"unicode-width",
]
[[package]]
name = "thiserror"
version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
version = "1.0.6" version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
[[package]]
name = "unicode-width"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.84" version = "0.2.84"
@ -351,34 +149,3 @@ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "xdg"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4583db5cbd4c4c0303df2d15af80f0539db703fa1c68802d4cbbd2dd0f88f6"
dependencies = [
"dirs",
]

View file

@ -21,6 +21,8 @@ exclude = [
[workspace] [workspace]
members = [ members = [
"examples/jotdown_wasm", "examples/jotdown_wasm",
]
exclude = [
"tests/afl", "tests/afl",
] ]

View file

@ -1,5 +1,30 @@
.POSIX: .POSIX:
all: jotdown docs
cargo build --workspace
jotdown: target/release/jotdown
cp $< $@
target/release/jotdown:
cargo build --release
.PHONY:
docs:
cargo doc --no-deps --workspace
.PHONY: lint
lint:
cargo fmt --all -- --check
cargo clippy -- -D warnings
cargo clippy --no-default-features -- -D warnings
cargo clippy --all-features -- -D warnings
.PHONY: check
check:
cargo test --workspace
cargo test --workspace --no-default-features
.PHONY: suite .PHONY: suite
suite: suite:
git submodule update --init modules/djot.js git submodule update --init modules/djot.js
@ -36,13 +61,20 @@ afl:
rm -rf tests/afl/out rm -rf tests/afl/out
(cd tests/afl && \ (cd tests/afl && \
cargo afl build --release --config profile.release.debug-assertions=true && \ cargo afl build --release --config profile.release.debug-assertions=true && \
(AFL_NO_UI=1 cargo afl fuzz -i in -o out -Mm ../../target/release/${AFL_TARGET} &) && \ (AFL_NO_UI=1 cargo afl fuzz -i in -o out -Mm target/release/${AFL_TARGET} &) && \
for i in $$(seq $$((${AFL_JOBS} - 1))); do \ for i in $$(seq $$((${AFL_JOBS} - 1))); do \
AFL_NO_UI=1 cargo afl fuzz -i in -o out -Ss$$i ../../target/release/${AFL_TARGET} & \ AFL_NO_UI=1 cargo afl fuzz -i in -o out -Ss$$i target/release/${AFL_TARGET} & \
done; \ done; \
trap - EXIT;\ trap - EXIT;\
cat) # keep process alive for trap cat) # keep process alive for trap
afl_quick:
rm -rf tests/afl/out
(cd tests/afl && \
cargo afl build --release --config profile.release.debug-assertions=true && \
AFL_NO_UI=1 AFL_BENCH_UNTIL_CRASH=1 \
cargo afl fuzz -i in -o out -V 60 target/release/${AFL_TARGET})
afl_crash: afl_crash:
set +e; \ set +e; \
for f in $$(find tests/afl/out -path '*/${AFL_TARGET_CRASH}/id*'); do \ for f in $$(find tests/afl/out -path '*/${AFL_TARGET_CRASH}/id*'); do \

View file

@ -27,6 +27,7 @@ pub fn valid<I: Iterator<Item = char>>(chars: I) -> (usize, bool) {
/// A collection of attributes, i.e. a key-value map. /// A collection of attributes, i.e. a key-value map.
// Attributes are relatively rare, we choose to pay 8 bytes always and sometimes an extra // Attributes are relatively rare, we choose to pay 8 bytes always and sometimes an extra
// indirection instead of always 24 bytes. // indirection instead of always 24 bytes.
#[allow(clippy::box_vec)]
#[derive(Debug, Clone, PartialEq, Eq, Default)] #[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct Attributes<'s>(Option<Box<Vec<(&'s str, CowStr<'s>)>>>); pub struct Attributes<'s>(Option<Box<Vec<(&'s str, CowStr<'s>)>>>);

View file

@ -351,13 +351,13 @@ impl<'s> TreeParser<'s> {
}); });
if let ListItem(ty) = c { if let ListItem(ty) = c {
if self let same_depth = self
.open_lists .open_lists
.last() .last()
.map_or(true, |OpenList { depth, .. }| { .map_or(true, |OpenList { depth, .. }| {
usize::from(*depth) < self.tree.depth() usize::from(*depth) < self.tree.depth()
}) });
{ if same_depth {
let tight = true; let tight = true;
let node = self.tree.enter( let node = self.tree.enter(
Node::Container(Container::List(ListKind { ty, tight })), Node::Container(Container::List(ListKind { ty, tight })),

View file

@ -442,11 +442,10 @@ impl<I: Iterator<Item = char> + Clone> Parser<I> {
if matches!(dir, Dir::Open) { if matches!(dir, Dir::Open) {
return None; return None;
} }
if matches!(dir, Dir::Both) let whitespace_after = self.events.back().map_or(false, |ev| {
&& self.events.back().map_or(false, |ev| {
matches!(ev.kind, EventKind::Whitespace | EventKind::Atom(Softbreak)) matches!(ev.kind, EventKind::Whitespace | EventKind::Atom(Softbreak))
}) });
{ if matches!(dir, Dir::Both) && whitespace_after {
return None; return None;
} }
@ -791,6 +790,7 @@ impl<I: Iterator<Item = char> + Clone> Iterator for Parser<I> {
type Item = Event; type Item = Event;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
#[allow(clippy::blocks_in_if_conditions)]
while self.events.is_empty() while self.events.is_empty()
|| !self.openers.is_empty() || !self.openers.is_empty()
|| self // for merge or attributes || self // for merge or attributes
@ -813,12 +813,13 @@ impl<I: Iterator<Item = char> + Clone> Iterator for Parser<I> {
EventKind::Str | EventKind::Whitespace => { EventKind::Str | EventKind::Whitespace => {
// merge str events // merge str events
let mut span = e.span; let mut span = e.span;
while self.events.front().map_or(false, |e| { let should_merge = |e: &Event, span: Span| {
matches!( matches!(
e.kind, e.kind,
EventKind::Str | EventKind::Whitespace | EventKind::Placeholder EventKind::Str | EventKind::Whitespace | EventKind::Placeholder
) && span.end() == e.span.start() ) && span.end() == e.span.start()
}) { };
while self.events.front().map_or(false, |e| should_merge(e, span)) {
let ev = self.events.pop_front().unwrap(); let ev = self.events.pop_front().unwrap();
span = span.union(ev.span); span = span.union(ev.span);
} }

View file

@ -241,7 +241,8 @@ pub enum Container<'s> {
impl<'s> Container<'s> { impl<'s> Container<'s> {
/// Is a block element. /// Is a block element.
fn is_block(&self) -> bool { #[must_use]
pub fn is_block(&self) -> bool {
match self { match self {
Self::Blockquote Self::Blockquote
| Self::List { .. } | Self::List { .. }
@ -278,7 +279,8 @@ impl<'s> Container<'s> {
} }
/// Is a block element that may contain children blocks. /// Is a block element that may contain children blocks.
fn is_block_container(&self) -> bool { #[must_use]
pub fn is_block_container(&self) -> bool {
match self { match self {
Self::Blockquote Self::Blockquote
| Self::List { .. } | Self::List { .. }

View file

@ -10,17 +10,19 @@ mod.rs: ${TEST_DJ} html
echo "use crate::suite_test;" > $@ echo "use crate::suite_test;" > $@
for name in ${TEST}; do \ for name in ${TEST}; do \
name_snake=$$(basename -a $$name | sed 's/-/_/g'); \ name_snake=$$(basename -a $$name | sed 's/-/_/g'); \
echo "#[test]" >> $@; \ skip_reason=$$(grep -E "^$${name_snake}:" skip | cut -d: -f2); \
echo "fn test_$$name_snake() {" >> $@; \ [ -n "$$skip_reason" ] && echo "#[ignore = \"$${skip_reason}\"]"; \
printf ' let src = r###"' >> $@; \ echo "#[test]"; \
cat $$name.dj >> $@; \ echo "fn $$name_snake() {"; \
echo '"###;' >> $@; \ printf ' let src = r###"'; \
printf ' let expected = r###"' >> $@; \ cat $$name.dj; \
cat $$name.html >> $@; \ echo '"###;'; \
echo '"###;' >> $@; \ printf ' let expected = r###"'; \
echo " suite_test!(src, expected);" >> $@; \ cat $$name.html; \
echo "}" >> $@; \ echo '"###;'; \
done echo " suite_test!(src, expected);"; \
echo "}"; \
done >> $@
html: djot-js ${TEST_DJ} html: djot-js ${TEST_DJ}
echo ${TEST} echo ${TEST}

3
tests/bench/skip Normal file
View file

@ -0,0 +1,3 @@
block_list_flat:large list marker number
inline_links_flat:escaped attributes, empty hrefs
inline_links_nested:empty link text

View file

@ -1,5 +1,7 @@
#[rustfmt::skip]
#[cfg(feature = "suite_bench")] #[cfg(feature = "suite_bench")]
mod bench; mod bench;
#[rustfmt::skip]
#[cfg(feature = "suite")] #[cfg(feature = "suite")]
mod suite; mod suite;

View file

@ -1,4 +1,6 @@
BEGIN { BEGIN {
FS=":"
while (getline < "skip") skips[$1]=$2
print "use crate::suite_test;" print "use crate::suite_test;"
print "" print ""
} }
@ -7,14 +9,16 @@ $0 ~ "^`{3,}$" {
l=length($0) l=length($0)
if (fence == 0) { # enter fence if (fence == 0) { # enter fence
print "#[test]" print "#[test]"
printf "fn test%02d() {\n", count
printf " let src = r##\""
fence=l fence=l
count+=1
} else if (fence == l) { # exit fence } else if (fence == l) { # exit fence
if (ignore) { if (ignore) {
ignore=0 ignore=0
} else { } else {
printf " let expected = r##\""
close("src")
while (getline l < "src") print l
close("src")
system("rm -f src")
print "\"##;" print "\"##;"
print " suite_test!(src, expected);" print " suite_test!(src, expected);"
print "}" print "}"
@ -22,7 +26,7 @@ $0 ~ "^`{3,}$" {
} }
fence=0 fence=0
} else { } else {
print $0 # md/html print $0 > "src" # md/html
} }
next next
} }
@ -34,14 +38,25 @@ fence == 0 && $0 ~ "^`{3,} .*$" {
} }
$0 ~ "^\\.$" && !ignore { # enter html $0 ~ "^\\.$" && !ignore { # enter html
close("src")
cmd="cat src | md5sum | cut -c-7"
cmd | getline hash
close(cmd)
if (hash in skips) printf "#[ignore = \"%s\"]\n", skips[hash]
printf "fn test_%s() {\n", hash
printf " let src = r##\""
while (getline l < "src") print l
close("src")
system("rm -f src")
print "\"##;" print "\"##;"
printf " let expected = r##\""
next next
} }
!ignore { !ignore {
if (fence==0 && $0 != "") { # comment if (fence) {
printf "// " # write to file so content can be piped to md5sum (without having to shell escape)
print $0 > "src" # md/html
} else if ($0 != "") {
printf "// %s\n", $0 # comment
} }
print $0 # comment/md/html
} }

20
tests/suite/skip Normal file
View file

@ -0,0 +1,20 @@
38d85f9:multi-line block attributes
6c14561:multi-line block attributes
48546bb:escape in attributes
6bc4257:escape in attributes
613a9d6:attribute container precedence
f4f22fc:attribute key class order
ae6fc15:bugged left/right quote
168469a:bugged left/right quote
2056174:unicode whitespace emph
2e8fffa:unicode whitespace strong
e1f5b5e:untrimmed whitespace before linebreak
07888f3:div close within raw block
8423412:heading id conflict with existing id
00a46ed:clear inline formatting from link tags
a8e17c3:empty href
c0a3dec:escape in url
e66af00:url container precedence
61876cf:roman alpha ambiguity
f31b357:roman alpha ambiguity
642d380:table end in verbatim inline