Merge pull request #2422 from pbalcer/bench-stability

[benchmarks] add stddev calculation and outlier elimination

Author: pbalcer

scripts/benchmarks/benches/base.py

@@ -71,9 +71,6 @@ def run(self, env_vars) -> list[Result]:
     def teardown(self):
         raise NotImplementedError()
 
-    def ignore_iterations(self):
-        return False
-
 class Suite:
     def benchmarks(self) -> list[Benchmark]:
         raise NotImplementedError()

scripts/benchmarks/benches/llamacpp.py

@@ -76,9 +76,6 @@ def name(self):
     def lower_is_better(self):
         return False
 
-    def ignore_iterations(self):
-        return True
-
     def run(self, env_vars) -> list[Result]:
         command = [
             f"{self.benchmark_bin}",

scripts/benchmarks/benches/options.py

@@ -15,13 +15,16 @@ class Options:
     rebuild: bool = True
     benchmark_cwd: str = "INVALID"
     timeout: float = 600
-    iterations: int = 5
+    iterations: int = 3
     verbose: bool = False
     compare: Compare = Compare.LATEST
     compare_max: int = 10 # average/median over how many results
     output_html: bool = False
     output_markdown: bool = True
     dry_run: bool = False
+    # these two should probably be merged into one setting
+    stddev_threshold: float = 0.02
+    epsilon: float = 0.02
 
 options = Options()
 

scripts/benchmarks/benches/result.py

@@ -23,6 +23,7 @@ class Result:
     lower_is_better: bool = True
     git_hash: str = ''
     date: Optional[datetime] = None
+    stddev: float = 0.0
 
 @dataclass_json
 @dataclass

scripts/benchmarks/main.py

@@ -18,10 +18,97 @@
 
 import argparse
 import re
+import statistics
 
 # Update this if you are changing the layout of the results files
 INTERNAL_WORKDIR_VERSION = '2.0'
 
+def run_iterations(benchmark: Benchmark, env_vars, iters: int, results: dict[str, list[Result]]):
+    for iter in range(iters):
+        print(f"running {benchmark.name()}, iteration {iter}... ", end='', flush=True)
+        bench_results = benchmark.run(env_vars)
+        if bench_results is None:
+            print(f"did not finish (OK for sycl-bench).")
+            break
+
+        for bench_result in bench_results:
+            # TODO: report failures in markdown/html ?
+            if not bench_result.passed:
+                print(f"complete ({bench_result.label}: verification FAILED)")
+                continue
+
+            print(f"complete ({bench_result.label}: {bench_result.value:.3f} {bench_result.unit}).")
+
+            bench_result.name = bench_result.label
+            bench_result.lower_is_better = benchmark.lower_is_better()
+
+            if bench_result.label not in results:
+                results[bench_result.label] = []
+
+            results[bench_result.label].append(bench_result)
+
+# https://www.statology.org/modified-z-score/
+def modified_z_score(values: list[float]) -> list[float]:
+    median = statistics.median(values)
+    mad = statistics.median([abs(v - median) for v in values])
+    if mad == 0:
+        return [0] * len(values)
+    return [(0.6745 * (v - median)) / mad for v in values]
+
+def remove_outliers(results: dict[str, list[Result]], threshold: float = 3.5) -> dict[str, list[Result]]:
+    new_results = {}
+    for key, rlist in results.items():
+        # don't eliminate outliers on first pass
+        if len(rlist) <= options.iterations:
+            new_results[key] = rlist
+            continue
+
+        values = [r.value for r in rlist]
+        z_scores = modified_z_score(values)
+        filtered_rlist = [r for r, z in zip(rlist, z_scores) if abs(z) <= threshold]
+
+        if not filtered_rlist:
+            new_results[key] = rlist
+        else:
+            new_results[key] = filtered_rlist
+
+    return new_results
+
+def process_results(results: dict[str, list[Result]]) -> tuple[bool, list[Result]]:
+    processed: list[Result] = []
+    # technically, we can detect whether result is below or above threshold per
+    # individual result. However, we can't repeat benchmark runs with that
+    # granularity. So we just reject all results and try again.
+    valid_results = True # above stddev threshold
+
+    for label, rlist in remove_outliers(results).items():
+        if (len(rlist) == 0):
+            continue
+
+        if len(rlist) == 1:
+            processed.append(rlist[0])
+            continue
+
+        values = [r.value for r in rlist]
+
+        mean_value = statistics.mean(values)
+        stddev = statistics.stdev(values)
+
+        threshold = options.stddev_threshold * mean_value
+
+        if stddev > threshold:
+            print(f"stddev {stddev} above the threshold {threshold} for {label}")
+            valid_results = False
+
+        rlist.sort(key=lambda res: res.value)
+        median_index = len(rlist) // 2
+        median_result = rlist[median_index]
+        median_result.stddev = stddev
+
+        processed.append(median_result)
+
+    return valid_results, processed
+
 def main(directory, additional_env_vars, save_name, compare_names, filter):
     prepare_workdir(directory, INTERNAL_WORKDIR_VERSION)
 
@@ -65,36 +152,15 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
     for benchmark in benchmarks:
         try:
             merged_env_vars = {**additional_env_vars}
-            iteration_results = []
-            iterations = options.iterations if not benchmark.ignore_iterations() else 1
-            for iter in range(iterations):
-                print(f"running {benchmark.name()}, iteration {iter}... ", end='', flush=True)
-                bench_results = benchmark.run(merged_env_vars)
-                if bench_results is not None:
-                    for bench_result in bench_results:
-                        if bench_result.passed:
-                            print(f"complete ({bench_result.label}: {bench_result.value:.3f} {bench_result.unit}).")
-                        else:
-                            print(f"complete ({bench_result.label}: verification FAILED)")
-                        iteration_results.append(bench_result)
-                else:
-                    print(f"did not finish (OK for sycl-bench).")
+            intermediate_results: dict[str, list[Result]] = {}
+            processed: list[Result] = []
+            for _ in range(5):
+                run_iterations(benchmark, merged_env_vars, options.iterations, intermediate_results)
+                valid, processed = process_results(intermediate_results)
+                if valid:
                     break
+            results += processed
 
-            if len(iteration_results) == 0:
-                continue
-
-            for label in set([result.label for result in iteration_results]):
-                label_results = [result for result in iteration_results if result.label == label and result.passed == True]
-                if len(label_results) > 0:
-                    label_results.sort(key=lambda res: res.value)
-                    median_index = len(label_results) // 2
-                    median_result = label_results[median_index]
-
-                    median_result.name = label
-                    median_result.lower_is_better = benchmark.lower_is_better()
-
-                    results.append(median_result)
         except Exception as e:
             if options.exit_on_failure:
                 raise e
@@ -164,14 +230,15 @@ def validate_and_parse_env_args(env_args):
     parser.add_argument("--env", type=str, help='Use env variable for a benchmark run.', action="append", default=[])
     parser.add_argument("--save", type=str, help='Save the results for comparison under a specified name.')
     parser.add_argument("--compare", type=str, help='Compare results against previously saved data.', action="append", default=["baseline"])
-    parser.add_argument("--iterations", type=int, help='Number of times to run each benchmark to select a median value.', default=5)
-    parser.add_argument("--timeout", type=int, help='Timeout for individual benchmarks in seconds.', default=600)
+    parser.add_argument("--iterations", type=int, help='Number of times to run each benchmark to select a median value.', default=options.iterations)
+    parser.add_argument("--stddev-threshold", type=float, help='If stddev % is above this threshold, rerun all iterations', default=options.stddev_threshold)
+    parser.add_argument("--timeout", type=int, help='Timeout for individual benchmarks in seconds.', default=options.timeout)
     parser.add_argument("--filter", type=str, help='Regex pattern to filter benchmarks by name.', default=None)
-    parser.add_argument("--epsilon", type=float, help='Threshold to consider change of performance significant', default=0.005)
+    parser.add_argument("--epsilon", type=float, help='Threshold to consider change of performance significant', default=options.epsilon)
     parser.add_argument("--verbose", help='Print output of all the commands.', action="store_true")
     parser.add_argument("--exit-on-failure", help='Exit on first failure.', action="store_true")
     parser.add_argument("--compare-type", type=str, choices=[e.value for e in Compare], help='Compare results against previously saved data.', default=Compare.LATEST.value)
-    parser.add_argument("--compare-max", type=int, help='How many results to read for comparisions', default=10)
+    parser.add_argument("--compare-max", type=int, help='How many results to read for comparisions', default=options.compare_max)
     parser.add_argument("--output-html", help='Create HTML output', action="store_true", default=False)
     parser.add_argument("--output-markdown", help='Create Markdown output', action="store_true", default=True)
     parser.add_argument("--dry-run", help='Do not run any actual benchmarks', action="store_true", default=False)