Add fuzzing for jsonrpc

Import the fuzzing from `jsonrpc`

source: https://github.com/apoelstra/rust-jsonrpc
commit hash: `59646e6e6ac95f07998133b1709e4a1fa2dbc7bd`
commit: `59646e6 Merge apoelstra/rust-jsonrpc#119: Use rust-bitcoin-maintainer-tools and re-write CI`

Then I update the `generate-files.sh` script to mimic a recent one from
`rust-bitcoin`.

Add a README explaining how we got to the current state.

Author: tcharding

.github/workflows/cron-daily-fuzz.yml

@@ -0,0 +1,62 @@
+# Automatically generated by fuzz/generate-files.sh
+name: Fuzz
+on:
+  schedule:
+    # 6am every day UTC, this correlates to:
+    # - 11pm PDT
+    # - 7am CET
+    # - 5pm AEDT
+    - cron: '00 06 * * *'
+
+jobs:
+  fuzz:
+    if: ${{ !github.event.act }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # We only get 20 jobs at a time, we probably don't want to go
+        # over that limit with fuzzing because of the hour run time.
+        fuzz_target: [
+          minreq_http,
+          simple_http,
+        ]
+    steps:
+      - name: Install test dependencies
+        run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        id: cache-fuzz
+        with:
+          path: |
+            ~/.cargo/bin
+            fuzz/target
+            target
+          key: cache-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: '1.65.0'
+      - name: fuzz
+        run: |
+          if [[ "${{ matrix.fuzz_target }}" =~ ^bitcoin ]]; then
+              export RUSTFLAGS='--cfg=hashes_fuzz --cfg=secp256k1_fuzz'
+          fi
+          echo "Using RUSTFLAGS $RUSTFLAGS"
+          cd fuzz && ./fuzz.sh "${{ matrix.fuzz_target }}"
+      - run: echo "${{ matrix.fuzz_target }}" >executed_${{ matrix.fuzz_target }}
+      - uses: actions/upload-artifact@v3
+        with:
+          name: executed_${{ matrix.fuzz_target }}
+          path: executed_${{ matrix.fuzz_target }}
+
+  verify-execution:
+    if: ${{ !github.event.act }}
+    needs: fuzz
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v3
+      - name: Display structure of downloaded files
+        run: ls -R
+      - run: find executed_* -type f -exec cat {} + | sort > executed
+      - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed

fuzz/.gitignore

@@ -0,0 +1,4 @@
+
+target
+corpus
+artifacts

fuzz/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "jsonrpc-fuzz"
+edition = "2021"
+rust-version = "1.63.0"
+version = "0.0.1"
+authors = ["Generated by fuzz/generate-files.sh"]
+publish = false
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+honggfuzz = { version = "0.5.55", default-features = false }
+jsonrpc = { path = "..", features = ["minreq_http"] }
+
+serde = { version = "1.0.103", features = [ "derive" ] }
+serde_json = "1.0"
+
+[lints.rust]
+unexpected_cfgs = { level = "deny", check-cfg = ['cfg(fuzzing)'] }
+
+[[bin]]
+name = "minreq_http"
+path = "fuzz_targets/minreq_http.rs"
+
+[[bin]]
+name = "simple_http"
+path = "fuzz_targets/simple_http.rs"

fuzz/README.md

@@ -0,0 +1,14 @@
+# `bitcoind-json-rpc` fuzzing
+
+Currently we only fuzz `jsonrpc`, the fuzz code was imported along
+with the `jsonrpc` import from `github.com/apoelstra/rust-jsonrpc`
+commit hash: `59646e6e6ac95f07998133b1709e4a1fa2dbc7bd` which is 
+
+commit: `59646e6 Merge apoelstra/rust-jsonrpc#119: Use rust-bitcoin-maintainer-tools and re-write CI`
+
+Then I updated the `generate-files.sh` script to mimic a recent one from `rust-bitcoin`.
+
+## Note to devs
+
+If you are considering adding fuzzing for the other crates take a look at how we set up
+`fuzz_target` in `rust-bitcoin/fuzz`.

fuzz/cycle.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Continuosly cycle over fuzz targets running each for 1 hour.
+# It uses chrt SCHED_IDLE so that other process takes priority.
+#
+# For hfuzz options see https://github.com/google/honggfuzz/blob/master/docs/USAGE.md
+
+set -e
+REPO_DIR=$(git rev-parse --show-toplevel)
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+while :
+do
+  for targetFile in $(listTargetFiles); do
+    targetName=$(targetFileToName "$targetFile")
+    echo "Fuzzing target $targetName ($targetFile)"
+
+    # fuzz for one hour
+    HFUZZ_RUN_ARGS='--run_time 3600' chrt -i 0 cargo hfuzz run "$targetName"
+    # minimize the corpus
+    HFUZZ_RUN_ARGS="-i hfuzz_workspace/$targetName/input/ -P -M" chrt -i 0 cargo hfuzz run "$targetName"
+  done
+done

fuzz/fuzz-util.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# Sort order is effected by locale. See `man sort`.
+# > Set LC_ALL=C to get the traditional sort order that uses native byte values.
+export LC_ALL=C
+
+listTargetFiles() {
+  pushd "$REPO_DIR/fuzz" > /dev/null || exit 1
+  find fuzz_targets/ -type f -name "*.rs" | sort
+  popd > /dev/null || exit 1
+}
+
+targetFileToName() {
+  echo "$1" \
+    | sed 's/^fuzz_targets\///' \
+    | sed 's/\.rs$//' \
+    | sed 's/\//_/g'
+}
+
+targetFileToHFuzzInputArg() {
+  baseName=$(basename "$1")
+  dirName="${baseName%.*}"
+  if [ -d "hfuzz_input/$dirName" ]; then
+    echo "HFUZZ_INPUT_ARGS=\"-f hfuzz_input/$FILE/input\""
+  fi
+}
+
+listTargetNames() {
+  for target in $(listTargetFiles); do
+    targetFileToName "$target"
+  done
+}
+
+# Utility function to avoid CI failures on Windows
+checkWindowsFiles() {
+  incorrectFilenames=$(find . -type f -name "*,*" -o -name "*:*" -o -name "*<*" -o -name "*>*" -o -name "*|*" -o -name "*\?*" -o -name "*\**" -o -name "*\"*" | wc -l)
+  if [ "$incorrectFilenames" -gt 0 ]; then
+    echo "Bailing early because there is a Windows-incompatible filename in the tree."
+    exit 2
+  fi
+}
+
+# Checks whether a fuzz case output some report, and dumps it in hex
+checkReport() {
+  reportFile="hfuzz_workspace/$1/HONGGFUZZ.REPORT.TXT"
+  if [ -f "$reportFile" ]; then
+    cat "$reportFile"
+    for CASE in "hfuzz_workspace/$1/SIG"*; do
+      xxd -p -c10000 < "$CASE"
+    done
+    exit 1
+  fi
+}

fuzz/fuzz.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -ex
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+# Check that input files are correct Windows file names
+checkWindowsFiles
+
+if [ "$1" == "" ]; then
+  targetFiles="$(listTargetFiles)"
+else
+  targetFiles=fuzz_targets/"$1".rs
+fi
+
+cargo --version
+rustc --version
+
+# Testing
+cargo install --force honggfuzz --no-default-features
+for targetFile in $targetFiles; do
+  targetName=$(targetFileToName "$targetFile")
+  echo "Fuzzing target $targetName ($targetFile)"
+  if [ -d "hfuzz_input/$targetName" ]; then
+    HFUZZ_INPUT_ARGS="-f hfuzz_input/$targetName/input\""
+  else
+    HFUZZ_INPUT_ARGS=""
+  fi
+  RUSTFLAGS="--cfg=jsonrpc_fuzz" HFUZZ_RUN_ARGS="--run_time 30 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run "$targetName"
+
+  checkReport "$targetName"
+done

fuzz/fuzz_targets/minreq_http.rs

@@ -0,0 +1,59 @@
+extern crate jsonrpc;
+
+// Note, tests are empty if "jsonrpc_fuzz" is not set but still show up in output of `cargo test --workspace`.
+
+#[allow(unused_variables)] // `data` is not used when "jsonrpc_fuzz" is not set.
+fn do_test(data: &[u8]) {
+    #[cfg(jsonrpc_fuzz)]
+    {
+        use std::io;
+
+        use jsonrpc::minreq_http::{MinreqHttpTransport, FUZZ_TCP_SOCK};
+        use jsonrpc::Client;
+
+        *FUZZ_TCP_SOCK.lock().unwrap() = Some(io::Cursor::new(data.to_vec()));
+
+        let t = MinreqHttpTransport::builder()
+            .url("localhost:123")
+            .expect("parse url")
+            .basic_auth("".to_string(), None)
+            .build();
+
+        let client = Client::with_transport(t);
+        let request = client.build_request("uptime", None);
+        let _ = client.send_request(request);
+    }
+}
+
+fn main() {
+    loop {
+        honggfuzz::fuzz!(|data| {
+            do_test(data);
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    fn extend_vec_from_hex(hex: &str) -> Vec<u8> {
+        let mut out = vec![];
+        let mut b = 0;
+        for (idx, c) in hex.as_bytes().iter().enumerate() {
+            b <<= 4;
+            match *c {
+                b'A'..=b'F' => b |= c - b'A' + 10,
+                b'a'..=b'f' => b |= c - b'a' + 10,
+                b'0'..=b'9' => b |= c - b'0',
+                _ => panic!("Bad hex"),
+            }
+            if (idx & 1) == 1 {
+                out.push(b);
+                b = 0;
+            }
+        }
+        out
+    }
+
+    #[test]
+    fn duplicate_crash() { super::do_test(&extend_vec_from_hex("00")); }
+}

fuzz/fuzz_targets/simple_http.rs

@@ -0,0 +1,59 @@
+extern crate jsonrpc;
+
+// Note, tests are if empty "jsonrpc_fuzz" is not set but still show up in output of `cargo test --workspace`.
+
+#[allow(unused_variables)] // `data` is not used when "jsonrpc_fuzz" is not set.
+fn do_test(data: &[u8]) {
+    #[cfg(jsonrpc_fuzz)]
+    {
+        use std::io;
+
+        use jsonrpc::simple_http::{SimpleHttpTransport, FUZZ_TCP_SOCK};
+        use jsonrpc::Client;
+
+        *FUZZ_TCP_SOCK.lock().unwrap() = Some(io::Cursor::new(data.to_vec()));
+
+        let t = SimpleHttpTransport::builder()
+            .url("localhost:123")
+            .expect("parse url")
+            .auth("", None)
+            .build();
+
+        let client = Client::with_transport(t);
+        let request = client.build_request("uptime", None);
+        let _ = client.send_request(request);
+    }
+}
+
+fn main() {
+    loop {
+        honggfuzz::fuzz!(|data| {
+            do_test(data);
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    fn extend_vec_from_hex(hex: &str) -> Vec<u8> {
+        let mut out = vec![];
+        let mut b = 0;
+        for (idx, c) in hex.as_bytes().iter().enumerate() {
+            b <<= 4;
+            match *c {
+                b'A'..=b'F' => b |= c - b'A' + 10,
+                b'a'..=b'f' => b |= c - b'a' + 10,
+                b'0'..=b'9' => b |= c - b'0',
+                _ => panic!("Bad hex"),
+            }
+            if (idx & 1) == 1 {
+                out.push(b);
+                b = 0;
+            }
+        }
+        out
+    }
+
+    #[test]
+    fn duplicate_crash() { super::do_test(&extend_vec_from_hex("00")); }
+}