From 49c1d8c753ac909f45b59f78bd93e579146265e4 Mon Sep 17 00:00:00 2001 From: Richard Wei Date: Sat, 16 Jul 2022 00:25:20 -0700 Subject: [PATCH] Add option to show benchmark charts --- Sources/RegexBenchmark/BenchmarkChart.swift | 112 +++++++++++++++++++ Sources/RegexBenchmark/BenchmarkRunner.swift | 30 ++++- Sources/RegexBenchmark/CLI.swift | 9 +- 3 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 Sources/RegexBenchmark/BenchmarkChart.swift diff --git a/Sources/RegexBenchmark/BenchmarkChart.swift b/Sources/RegexBenchmark/BenchmarkChart.swift new file mode 100644 index 000000000..3bb8e60ad --- /dev/null +++ b/Sources/RegexBenchmark/BenchmarkChart.swift @@ -0,0 +1,112 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021-2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if os(macOS) + +import Charts +import SwiftUI + +struct BenchmarkChart: View { + struct Comparison: Identifiable { + var id = UUID() + var name: String + var baseline: BenchmarkResult + var latest: BenchmarkResult + } + + var comparisons: [Comparison] + + var body: some View { + VStack(alignment: .leading) { + ForEach(comparisons) { comparison in + let new = comparison.latest.median.seconds + let old = comparison.baseline.median.seconds + Chart { + chartBody( + name: comparison.name, + new: new, + old: old, + sampleCount: comparison.latest.samples) + } + .chartXAxis { + AxisMarks { value in + AxisTick() + AxisValueLabel { + Text(String(format: "%.5fs", value.as(Double.self)!)) + } + } + } + .chartYAxis { + AxisMarks { value in + AxisGridLine() + AxisValueLabel { + HStack { + Text(value.as(String.self)!) + let delta = (new - old) / old * 100 + Text(String(format: "%+.2f%%", delta)) + .foregroundColor(delta <= 0 ? .green : .yellow) + } + } + } + } + .frame(idealHeight: 60) + } + } + } + + @ChartContentBuilder + func chartBody( + name: String, + new: TimeInterval, + old: TimeInterval, + sampleCount: Int + ) -> some ChartContent { + // Baseline bar + BarMark( + x: .value("Time", old), + y: .value("Name", "\(name) (\(sampleCount) samples)")) + .position(by: .value("Kind", "Baseline")) + .foregroundStyle(.gray) + + // Latest result bar + BarMark( + x: .value("Time", new), + y: .value("Name", "\(name) (\(sampleCount) samples)")) + .position(by: .value("Kind", "Latest")) + .foregroundStyle(LinearGradient( + colors: [.accentColor, new - old <= 0 ? .green : .yellow], + startPoint: .leading, + endPoint: .trailing)) + + // Comparison + RuleMark(x: .value("Time", new)) + .foregroundStyle(.gray) + .lineStyle(.init(lineWidth: 0.5, dash: [2])) + } +} + +struct BenchmarkResultApp: App { + static var comparisons: [BenchmarkChart.Comparison]? + + var body: some Scene { + WindowGroup { + if let comparisons = Self.comparisons { + ScrollView { + BenchmarkChart(comparisons: comparisons) + } + } else { + Text("No data") + } + } + } +} + +#endif diff --git a/Sources/RegexBenchmark/BenchmarkRunner.swift b/Sources/RegexBenchmark/BenchmarkRunner.swift index 78953bdd6..93fbdbe23 100644 --- a/Sources/RegexBenchmark/BenchmarkRunner.swift +++ b/Sources/RegexBenchmark/BenchmarkRunner.swift @@ -89,7 +89,7 @@ extension BenchmarkRunner { try results.save(to: url) } - func compare(against compareFilePath: String) throws { + func compare(against compareFilePath: String, showChart: Bool) throws { let compareFileURL = URL(fileURLWithPath: compareFilePath) let compareResult = try SuiteResult.load(from: compareFileURL) let compareFile = compareFileURL.lastPathComponent @@ -121,6 +121,32 @@ extension BenchmarkRunner { for item in improvements { printComparison(name: item.key, diff: item.value) } + + #if os(macOS) + if showChart { + print(""" + === Comparison chart ================================================================= + Press Control-C to close... + """) + BenchmarkResultApp.comparisons = { + var comparisons: [BenchmarkChart.Comparison] = [] + for (name, baseline) in compareResult.results { + if let latest = results.results[name] { + comparisons.append( + .init(name: name, baseline: baseline, latest: latest)) + } + } + return comparisons.sorted { + let delta0 = Float($0.latest.median.seconds - $0.baseline.median.seconds) + / Float($0.baseline.median.seconds) + let delta1 = Float($1.latest.median.seconds - $1.baseline.median.seconds) + / Float($1.baseline.median.seconds) + return delta0 > delta1 + } + }() + BenchmarkResultApp.main() + } + #endif } } @@ -128,7 +154,7 @@ struct BenchmarkResult: Codable { let median: Time let stdev: Double let samples: Int - + init(_ median: Time, _ stdev: Double, _ samples: Int) { self.median = median self.stdev = stdev diff --git a/Sources/RegexBenchmark/CLI.swift b/Sources/RegexBenchmark/CLI.swift index 8ef351329..8afe9144a 100644 --- a/Sources/RegexBenchmark/CLI.swift +++ b/Sources/RegexBenchmark/CLI.swift @@ -17,6 +17,9 @@ struct Runner: ParsableCommand { @Option(help: "The result file to compare against") var compare: String? + @Flag(help: "Show comparison chart") + var showChart: Bool = false + @Flag(help: "Quiet mode") var quiet = false @@ -40,12 +43,12 @@ struct Runner: ParsableCommand { runner.suite = runner.suite.filter { b in !b.name.contains("NS") } } runner.run() - if let compareFile = compare { - try runner.compare(against: compareFile) - } if let saveFile = save { try runner.save(to: saveFile) } + if let compareFile = compare { + try runner.compare(against: compareFile, showChart: showChart) + } } } }