Skip to content

Commit a215876

Browse files
2bndy52bndy5
committed
add CI to detect performance regressions
Compares two release builds of cpp-linter binary: 1. the previous commit (for push events) or the base branch of a PR 2. the newest commit on the branch 3. the latest v1.x release of the pure-python cpp-linter package Caching is enabled to reduce CI runtime. Results are output to the CI workflow's job summary. This CI does not (currently) fail when a regression is detected. pin
1 parent 023c170 commit a215876

File tree

2 files changed

+181
-0
lines changed

2 files changed

+181
-0
lines changed

‎.github/workflows/perf-test.yml

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
name: Performance Regression
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- cpp-linter/src/**
8+
- cpp-linter/Cargo.toml
9+
- Cargo.toml
10+
- Cargo.lock
11+
- .github/workflows/perf-test.yml
12+
- .github/workflows/bench.py
13+
tags-ignore: ['*']
14+
pull_request:
15+
branches: [main]
16+
paths:
17+
- cpp-linter/src/**
18+
- cpp-linter/Cargo.toml
19+
- Cargo.toml
20+
- Cargo.lock
21+
- .github/workflows/perf-test.yml
22+
- .github/workflows/bench.py
23+
24+
jobs:
25+
build:
26+
name: Build ${{ matrix.name }}
27+
runs-on: ubuntu-latest
28+
strategy:
29+
matrix:
30+
include:
31+
- commit: ${{ github.sha }}
32+
name: current
33+
- commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
34+
name: previous
35+
steps:
36+
- name: Checkout ${{ matrix.name }}
37+
uses: actions/checkout@v4
38+
with:
39+
ref: ${{ matrix.commit }}
40+
- name: Cache base ref build
41+
uses: actions/cache@v4
42+
id: cache
43+
with:
44+
key: bin-cache-${{ matrix.name }}-${{ matrix.commit }}
45+
path: target/release/cpp-linter
46+
- run: rustup update --no-self-update
47+
if: steps.cache.outputs.cache-hit != 'true'
48+
- run: cargo build --bin cpp-linter --release
49+
if: steps.cache.outputs.cache-hit != 'true'
50+
- name: Upload build artifact
51+
uses: actions/upload-artifact@v4
52+
with:
53+
name: ${{ matrix.name }}
54+
path: target/release/cpp-linter
55+
56+
benchmark:
57+
name: Measure Performance Difference
58+
needs: [build]
59+
runs-on: ubuntu-latest
60+
steps:
61+
- uses: actions/checkout@v4
62+
- name: Checkout libgit2
63+
uses: actions/checkout@v4
64+
with:
65+
repository: libgit2/libgit2
66+
ref: v1.8.1
67+
path: libgit2
68+
- name: Download built binaries
69+
uses: actions/download-artifact@v4
70+
- name: Make binaries executable
71+
run: chmod +x ./*/cpp-linter
72+
- name: Generate compilation database
73+
working-directory: libgit2
74+
run: |
75+
mkdir build && cd build
76+
cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
77+
- name: Install cargo-binstall
78+
uses: cargo-bins/cargo-binstall@main
79+
- name: Install hyperfine
80+
run: cargo binstall -y hyperfine
81+
- uses: actions/setup-python@v5
82+
with:
83+
python-version: 3.x
84+
- run: pip install cpp-linter==1.10.2
85+
- name: Warmup and list files
86+
env:
87+
CPP_LINTER_COLOR: true
88+
working-directory: libgit2
89+
# Use previous build for stability. This will
90+
# - create the .cpp-linter_cache folder
91+
# - list the files concerning the benchmark test
92+
# NOTE: This does not actually invoke clang tools.
93+
run: ../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -s="" -c="-*" -e c
94+
- name: Run hyperfine tool
95+
# using the generated compilation database,
96+
# we will use cpp-linter (both builds) to scan libgit2 src/libgit2/**.c files.
97+
working-directory: libgit2
98+
run: >-
99+
hyperfine
100+
--runs 2
101+
--style color
102+
--export-markdown '${{ runner.temp }}/benchmark.md'
103+
--export-json '${{ runner.temp }}/benchmark.json'
104+
--command-name=previous-build
105+
"../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -e c"
106+
--command-name=current-build
107+
"../current/cpp-linter -l 0 -p build -i='|!src/libgit2' -e c"
108+
--command-name=pure-python
109+
"cpp-linter -l false -j 0 -p build -i='|!src/libgit2' -e c"
110+
- name: Append report to job summary
111+
run: cat ${{ runner.temp }}/benchmark.md >> $GITHUB_STEP_SUMMARY
112+
- name: Upload JSON results
113+
uses: actions/upload-artifact@v4
114+
with:
115+
path: ${{ runner.temp }}/benchmark.json
116+
- name: Annotate summary
117+
run: python .github/workflows/perf_annotate.py "${{ runner.temp }}/benchmark.json"

‎.github/workflows/perf_annotate.py

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import argparse
2+
import json
3+
from os import environ
4+
from pathlib import Path
5+
from typing import List, Any, Dict
6+
7+
8+
class Args(argparse.Namespace):
9+
json_file: Path
10+
11+
12+
def main():
13+
arg_parser = argparse.ArgumentParser()
14+
arg_parser.add_argument("json_file", type=Path)
15+
arg_parser.parse_args(namespace=Args)
16+
17+
bench_json = Args.json_file.read_text(encoding="utf-8")
18+
bench: List[Dict[str, Any]] = json.loads(bench_json)["results"]
19+
20+
assert len(bench) == 3
21+
assert bench[0]["command"] == "previous-build"
22+
assert bench[1]["command"] == "current-build"
23+
assert bench[2]["command"] == "pure-python"
24+
25+
old_mean: float = bench[0]["mean"]
26+
new_mean: float = bench[1]["mean"]
27+
28+
diff = round(new_mean - old_mean, 2)
29+
scalar = round(new_mean / old_mean, 2) * 100
30+
31+
output = []
32+
if diff > 2:
33+
output.extend(
34+
[
35+
"> [!CAUTION]",
36+
"> Detected a performance regression in new changes:",
37+
]
38+
)
39+
elif diff < -2:
40+
output.extend(
41+
[
42+
"> [!TIP]",
43+
"> Detected a performance improvement in new changes:",
44+
]
45+
)
46+
else:
47+
output.extend(
48+
[
49+
"> [!NOTE]",
50+
"> Determined a negligible difference in performance with new changes:",
51+
]
52+
)
53+
output[-1] += f" {diff}s ({scalar} %)"
54+
annotation = "\n".join(output)
55+
56+
if "GITHUB_STEP_SUMMARY" in environ:
57+
with open(environ["GITHUB_STEP_SUMMARY"], "a") as summary:
58+
summary.write(f"\n{annotation}\n")
59+
else:
60+
print(annotation)
61+
62+
63+
if __name__ == "__main__":
64+
main()

0 commit comments

Comments
 (0)