Skip to content

Commit d7484f6

Browse files
committed
add CI to detect performance regressions
Compares two release builds of cpp-linter binary: 1. the previous commit (for push events) or the base branch of a PR 2. the newest commit on the branch 3. the latest v1.x release of the pure-python cpp-linter package Caching is enabled to reduce CI runtime. Results are output to the CI workflow's job summary. This CI does not (currently) fail when a regression is detected.
1 parent 023c170 commit d7484f6

File tree

2 files changed

+199
-0
lines changed

2 files changed

+199
-0
lines changed

.github/workflows/perf-test.yml

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
name: Performance Regression
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- cpp-linter/src/**
8+
- cpp-linter/Cargo.toml
9+
- Cargo.toml
10+
- Cargo.lock
11+
- .github/workflows/perf-test.yml
12+
- .github/workflows/bench.py
13+
tags-ignore: ['*']
14+
pull_request:
15+
branches: [main]
16+
paths:
17+
- cpp-linter/src/**
18+
- cpp-linter/Cargo.toml
19+
- Cargo.toml
20+
- Cargo.lock
21+
- .github/workflows/perf*
22+
jobs:
23+
build:
24+
name: Build ${{ matrix.name }}
25+
runs-on: ubuntu-latest
26+
strategy:
27+
matrix:
28+
include:
29+
- commit: ${{ github.sha }}
30+
name: current
31+
- commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
32+
name: previous
33+
outputs:
34+
cached-previous: ${{ steps.is-cached-previous.outputs.is-cached }}
35+
cached-current: ${{ steps.is-cached-current.outputs.is-cached }}
36+
steps:
37+
- name: Checkout ${{ matrix.name }}
38+
uses: actions/checkout@v4
39+
with:
40+
ref: ${{ matrix.commit }}
41+
- name: Cache base ref build
42+
uses: actions/cache@v4
43+
id: cache
44+
with:
45+
key: bin-cache-${{ hashFiles('cpp-linter/src/**', 'Cargo.toml', 'Cargo.lock', 'cpp-linter/Cargo.toml') }}
46+
path: target/release/cpp-linter
47+
- name: Is previous cached?
48+
if: matrix.name == 'previous'
49+
id: is-cached-previous
50+
run: echo "is-cached=${{ steps.cache.outputs.cache-hit }}" >> $GITHUB_OUTPUT
51+
- name: Is current cached?
52+
if: matrix.name == 'current'
53+
id: is-cached-current
54+
run: echo "is-cached=${{ steps.cache.outputs.cache-hit }}" >> $GITHUB_OUTPUT
55+
- run: rustup update --no-self-update
56+
if: steps.cache.outputs.cache-hit != 'true'
57+
- run: cargo build --bin cpp-linter --release
58+
if: steps.cache.outputs.cache-hit != 'true'
59+
- name: Upload build artifact
60+
uses: actions/upload-artifact@v4
61+
with:
62+
name: ${{ matrix.name }}
63+
path: target/release/cpp-linter
64+
65+
benchmark:
66+
name: Measure Performance Difference
67+
needs: [build]
68+
if: ${{ needs.build.outputs.cached-current != 'true' || needs.build.outputs.cached-previous != 'true' }}
69+
runs-on: ubuntu-latest
70+
steps:
71+
- uses: actions/checkout@v4
72+
- name: Checkout libgit2
73+
uses: actions/checkout@v4
74+
with:
75+
repository: libgit2/libgit2
76+
ref: v1.8.1
77+
path: libgit2
78+
- name: Download built binaries
79+
uses: actions/download-artifact@v4
80+
- name: Make binaries executable
81+
run: chmod +x ./*/cpp-linter
82+
- name: Generate compilation database
83+
working-directory: libgit2
84+
run: |
85+
mkdir build && cd build
86+
cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
87+
- name: Install cargo-binstall
88+
uses: cargo-bins/cargo-binstall@main
89+
- name: Install hyperfine
90+
run: cargo binstall -y hyperfine
91+
- uses: actions/setup-python@v5
92+
with:
93+
python-version: 3.x
94+
- run: pip install 'cpp-linter < 2.0'
95+
- name: Warmup and list files
96+
env:
97+
CPP_LINTER_COLOR: true
98+
working-directory: libgit2
99+
# Use previous build for stability. This will
100+
# - create the .cpp-linter_cache folder
101+
# - list the files concerning the benchmark test
102+
# NOTE: This does not actually invoke clang tools.
103+
run: ../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -s="" -c="-*" -e c
104+
- name: Run hyperfine tool
105+
# using the generated compilation database,
106+
# we will use cpp-linter (both builds) to scan libgit2 src/libgit2/**.c files.
107+
working-directory: libgit2
108+
run: >-
109+
hyperfine
110+
--runs 2
111+
--style color
112+
--export-markdown '${{ runner.temp }}/benchmark.md'
113+
--export-json '${{ runner.temp }}/benchmark.json'
114+
--command-name=previous-build
115+
"../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -e c"
116+
--command-name=current-build
117+
"../current/cpp-linter -l 0 -p build -i='|!src/libgit2' -e c"
118+
--command-name=pure-python
119+
"cpp-linter -l false -j 0 -p build -i='|!src/libgit2' -e c"
120+
- name: Append report to job summary
121+
run: cat ${{ runner.temp }}/benchmark.md >> $GITHUB_STEP_SUMMARY
122+
- name: Upload JSON results
123+
uses: actions/upload-artifact@v4
124+
with:
125+
name: benchmark-json
126+
path: ${{ runner.temp }}/benchmark.json
127+
- name: Annotate summary
128+
run: python .github/workflows/perf_annotate.py "${{ runner.temp }}/benchmark.json"
129+
130+
report-no-src-changes:
131+
runs-on: ubuntu-latest
132+
needs: [build]
133+
if: needs.build.outputs.cached-current == 'true' && needs.build.outputs.cached-previous == 'true'
134+
steps:
135+
- run: echo "::notice title=No benchmark performed::No changes to cpp-linter source code detected."

.github/workflows/perf_annotate.py

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import argparse
2+
import json
3+
from os import environ
4+
from pathlib import Path
5+
from typing import List, Any, Dict
6+
7+
8+
class Args(argparse.Namespace):
9+
json_file: Path
10+
11+
12+
def main():
13+
arg_parser = argparse.ArgumentParser()
14+
arg_parser.add_argument("json_file", type=Path)
15+
arg_parser.parse_args(namespace=Args)
16+
17+
bench_json = Args.json_file.read_text(encoding="utf-8")
18+
bench: List[Dict[str, Any]] = json.loads(bench_json)["results"]
19+
20+
assert len(bench) == 3
21+
assert bench[0]["command"] == "previous-build"
22+
assert bench[1]["command"] == "current-build"
23+
assert bench[2]["command"] == "pure-python"
24+
25+
old_mean: float = bench[0]["mean"]
26+
new_mean: float = bench[1]["mean"]
27+
28+
diff = round(new_mean - old_mean, 2)
29+
scalar = round(new_mean / old_mean, 2) * 100
30+
31+
output = []
32+
if diff > 2:
33+
output.extend(
34+
[
35+
"> [!CAUTION]",
36+
"> Detected a performance regression in new changes:",
37+
]
38+
)
39+
elif diff < -2:
40+
output.extend(
41+
[
42+
"> [!TIP]",
43+
"> Detected a performance improvement in new changes:",
44+
]
45+
)
46+
else:
47+
output.extend(
48+
[
49+
"> [!NOTE]",
50+
"> Determined a negligible difference in performance with new changes:",
51+
]
52+
)
53+
output[-1] += f" {diff}s ({scalar} %)"
54+
annotation = "\n".join(output)
55+
56+
if "GITHUB_STEP_SUMMARY" in environ:
57+
with open(environ["GITHUB_STEP_SUMMARY"], "a") as summary:
58+
summary.write(f"\n{annotation}\n")
59+
else:
60+
print(annotation)
61+
62+
63+
if __name__ == "__main__":
64+
main()

0 commit comments

Comments
 (0)