add CI to detect performance regressions

Compares two release builds of cpp-linter binary:
1. the previous commit (for push events) or the base branch of a PR
2. the newest commit on the branch

Caching is enabled to reduce CI runtime.

Results are output to the CI workflow's job summary.
This CI does not (currently) fail when a regression is detected.

Author: 2bndy5

.github/workflows/perf-test.yml

@@ -0,0 +1,114 @@
+name: Performance Regression
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - cpp-linter/src/**
+      - cpp-linter/Cargo.toml
+      - Cargo.toml
+      - Cargo.lock
+      - .github/workflows/perf-test.yml
+      - .github/workflows/bench.py
+    tags-ignore: ['*']
+  pull_request:
+    branches: [main]
+    paths:
+      - cpp-linter/src/**
+      - cpp-linter/Cargo.toml
+      - Cargo.toml
+      - Cargo.lock
+      - .github/workflows/perf-test.yml
+      - .github/workflows/bench.py
+
+jobs:
+  build:
+    name: Build ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - commit: ${{ github.sha }}
+            name: current
+          - commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
+            name: previous
+    steps:
+      - name: Checkout ${{ matrix.name }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ matrix.commit }}
+      - name: Cache base ref build
+        uses: actions/cache@v4
+        id: cache
+        with:
+          key: bin-cache-${{ matrix.name }}-${{ matrix.commit }}
+          path: target/release/cpp-linter
+      - run: rustup update --no-self-update
+        if: steps.cache.outputs.cache-hit != 'true'
+      - run: cargo build --bin cpp-linter --release
+        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Upload build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.name }}
+          path: target/release/cpp-linter
+
+  benchmark:
+    name: Measure Performance Difference
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Checkout libgit2
+        uses: actions/checkout@v4
+        with:
+          repository: libgit2/libgit2
+          ref: v1.8.1
+          path: libgit2
+      - name: Download built binaries
+        uses: actions/download-artifact@v4
+      - name: Make binaries executable
+        run: chmod +x ./*/cpp-linter
+      - name: Generate compilation database
+        working-directory: libgit2
+        run: |
+          mkdir build && cd build
+          cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
+      - name: Install cargo-binstall
+        uses: cargo-bins/cargo-binstall@main
+      - name: Install hyperfine
+        run: cargo binstall -y hyperfine
+      - name: Warmup and list files
+        env:
+          CPP_LINTER_COLOR: true
+        working-directory: libgit2
+        # Use previous build for stability. This will
+        # - create the .cpp-linter_cache folder
+        # - list the files concerning the benchmark test
+        # NOTE: This does not actually invoke clang tools.
+        run: ../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -s="" -c="-*"
+      - name: Run hyperfine tool
+        # using the generated compilation database,
+        # we will use cpp-linter (both builds) to scan libgit2 sources.
+        working-directory: libgit2
+        run: >-
+          hyperfine
+          --runs 1
+          --style color
+          --export-markdown '${{ runner.temp }}/benchmark.md'
+          --export-json '${{ runner.temp }}/benchmark.json'
+          --command-name=previous-build
+          "../previous/cpp-linter -l 0 -p build -i='|!src/libgit2'"
+          --command-name=current-build
+          "../current/cpp-linter -l 0 -p build -i='|!src/libgit2'"
+      - name: Append report to job summary
+        run: cat ${{ runner.temp }}/benchmark.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload JSON results
+        uses: actions/upload-artifact@v4
+        with:
+          path: ${{ runner.temp }}/benchmark.json
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Annotate summary
+        run: python .github/workflows/perf_annotate.py "${{ runner.temp }}/benchmark.json"

.github/workflows/perf_annotate.py

@@ -0,0 +1,63 @@
+import argparse
+import json
+from os import environ
+from pathlib import Path
+from typing import List, Any, Dict
+
+
+class Args(argparse.Namespace):
+    json_file: Path
+
+
+def main():
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument("json_file", type=Path)
+    arg_parser.parse_args(namespace=Args)
+
+    bench_json = Args.json_file.read_text(encoding="utf-8")
+    bench: List[Dict[str, Any]] = json.loads(bench_json)["results"]
+
+    assert len(bench) == 2
+    assert bench[0]["command"] == "previous-build"
+    assert bench[1]["command"] == "current-build"
+
+    old_mean: float = bench[0]["mean"]
+    new_mean: float = bench[1]["mean"]
+
+    diff = round(new_mean - old_mean, 2)
+    scalar = round(new_mean / old_mean, 2) * 100
+
+    output = []
+    if diff > 2:
+        output.extend(
+            [
+                "> [!CAUTION]",
+                "> Detected a performance regression in new changes:",
+            ]
+        )
+    elif diff < -2:
+        output.extend(
+            [
+                "> [!TIP]",
+                "> Detected a performance improvement in new changes:",
+            ]
+        )
+    else:
+        output.extend(
+            [
+                "> [!NOTE]",
+                "> Determined a negligible difference in performance with new changes:",
+            ]
+        )
+    output[-1] += f" {diff}s ({scalar} %)"
+    annotation = "\n".join(output)
+
+    if "GITHUB_STEP_SUMMARY" in environ:
+        with open(environ["GITHUB_STEP_SUMMARY"], "a") as summary:
+            summary.write(f"\n{annotation}\n")
+    else:
+        print(annotation)
+
+
+if __name__ == "__main__":
+    main()