Skip to content

Commit 1c8d410

Browse files
davidmorgandavidmorgan
committed
Add benchmark tool.
1 parent 5ce397c commit 1c8d410

12 files changed

+703
-0
lines changed

‎_benchmark/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Benchmarks `build_runner` against synthetic codebases applying real generators:
2+
`built_value`, `freezed`, `json_serializable` or `mockito`.
3+
4+
Example usage:
5+
6+
```
7+
dart run _benchmark --generator=built_value benchmark
8+
```

‎_benchmark/bin/_benchmark.dart

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:io';
6+
7+
import 'package:_benchmark/commands.dart';
8+
import 'package:_benchmark/generators.dart';
9+
import 'package:args/command_runner.dart';
10+
11+
final commandRunner =
12+
CommandRunner<void>(
13+
'dart run _benchmark',
14+
'Benchmarks build_runner performance.',
15+
)
16+
..addCommand(BenchmarkCommand())
17+
..addCommand(MeasureCommand())
18+
..addCommand(CreateCommand())
19+
..argParser.addOption(
20+
'generator',
21+
help: 'Generator to benchmark.',
22+
allowed: Generator.values.map((e) => e.packageName).toList(),
23+
defaultsTo: Generator.builtValue.packageName,
24+
)
25+
..argParser.addOption(
26+
'root-directory',
27+
help: 'Root directory for generated source and builds.',
28+
defaultsTo: '${Directory.systemTemp.path}/build_benchmark',
29+
);
30+
31+
Future<void> main(List<String> arguments) async {
32+
await commandRunner.run(arguments);
33+
}

‎_benchmark/lib/benchmark.dart

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'config.dart';
6+
7+
abstract interface class Benchmark {
8+
void create(RunConfig config);
9+
}
10+
11+
class Benchmarks {
12+
static String libraryName(int libraryNumber, {required int benchmarkSize}) {
13+
// Start numbering from 1.
14+
++libraryNumber;
15+
// Pad with zeros so alphabetic sort gives numerical ordering.
16+
final sizeDigits = benchmarkSize.toString().length;
17+
return 'lib${libraryNumber.toString().padLeft(sizeDigits, '0')}.dart';
18+
}
19+
20+
static String partName(
21+
int libraryNumber, {
22+
required int benchmarkSize,
23+
String infix = 'g',
24+
}) => libraryName(
25+
libraryNumber,
26+
benchmarkSize: benchmarkSize,
27+
).replaceAll('.dart', '.$infix.dart');
28+
29+
static String testName(int testNumber, {required int benchmarkSize}) {
30+
// Start numbering from 1.
31+
++testNumber;
32+
// Pad with zeros so alphabetic sort gives numerical ordering.
33+
final sizeDigits = benchmarkSize.toString().length;
34+
return 'some_test${testNumber.toString().padLeft(sizeDigits, '0')}.dart';
35+
}
36+
}

‎_benchmark/lib/benchmarks/built_value_generator_benchmark.dart

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import '../benchmark.dart';
6+
import '../config.dart';
7+
8+
class BuiltValueGeneratorBenchmark implements Benchmark {
9+
const BuiltValueGeneratorBenchmark();
10+
11+
@override
12+
void create(RunConfig config) {
13+
final workspace = config.workspace;
14+
final size = config.size;
15+
16+
// TODO(davidmorgan): add a way to pick `build` and generator versions.
17+
workspace.write(
18+
'pubspec.yaml',
19+
source: '''
20+
name: ${workspace.name}
21+
publish_to: none
22+
23+
environment:
24+
sdk: ^3.6.0
25+
26+
dependencies:
27+
built_value: any
28+
29+
dev_dependencies:
30+
build_runner: any
31+
built_value_generator: any
32+
''',
33+
);
34+
35+
final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
36+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
37+
final libraryName = Benchmarks.libraryName(
38+
libraryNumber,
39+
benchmarkSize: size,
40+
);
41+
appLines.add("import '$libraryName';");
42+
}
43+
workspace.write(
44+
'lib/app.dart',
45+
source: appLines.map((l) => '$l\n').join(''),
46+
);
47+
48+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
49+
final libraryName = Benchmarks.libraryName(
50+
libraryNumber,
51+
benchmarkSize: size,
52+
);
53+
final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size);
54+
workspace.write(
55+
'lib/$libraryName',
56+
source: '''
57+
// ignore_for_file: unused_import
58+
import 'package:built_value/built_value.dart';
59+
60+
import 'app.dart';
61+
62+
part '$partName';
63+
64+
abstract class Value implements Built<Value, ValueBuilder> {
65+
Value._();
66+
factory Value(void Function(ValueBuilder) updates) = _\$Value;
67+
}
68+
''',
69+
);
70+
}
71+
}
72+
}

‎_benchmark/lib/benchmarks/freezed_generator_benchmark.dart

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import '../benchmark.dart';
6+
import '../config.dart';
7+
8+
class FreezedGeneratorBenchmark implements Benchmark {
9+
const FreezedGeneratorBenchmark();
10+
11+
@override
12+
void create(RunConfig config) {
13+
final workspace = config.workspace;
14+
final size = config.size;
15+
16+
// TODO(davidmorgan): add a way to pick `build` and generator versions.
17+
workspace.write(
18+
'pubspec.yaml',
19+
source: '''
20+
name: ${workspace.name}
21+
publish_to: none
22+
23+
environment:
24+
sdk: ^3.6.0
25+
26+
dependencies:
27+
freezed_annotation: any
28+
29+
dev_dependencies:
30+
build_runner: any
31+
freezed: any
32+
''',
33+
);
34+
35+
final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
36+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
37+
final libraryName = Benchmarks.libraryName(
38+
libraryNumber,
39+
benchmarkSize: size,
40+
);
41+
appLines.add("import '$libraryName';");
42+
}
43+
workspace.write(
44+
'lib/app.dart',
45+
source: appLines.map((l) => '$l\n').join(''),
46+
);
47+
48+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
49+
final libraryName = Benchmarks.libraryName(
50+
libraryNumber,
51+
benchmarkSize: size,
52+
);
53+
final partName = Benchmarks.partName(
54+
libraryNumber,
55+
benchmarkSize: size,
56+
infix: 'freezed',
57+
);
58+
workspace.write(
59+
'lib/$libraryName',
60+
source: '''
61+
// ignore_for_file: unused_import
62+
import 'package:freezed_annotation/freezed_annotation.dart';
63+
64+
import 'app.dart';
65+
66+
part '$partName';
67+
68+
@freezed
69+
class Value with _\$Value {
70+
const factory Value() = _Value;
71+
}
72+
''',
73+
);
74+
}
75+
}
76+
}

‎_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import '../benchmark.dart';
6+
import '../config.dart';
7+
8+
class JsonSerializableGeneratorBenchmark implements Benchmark {
9+
const JsonSerializableGeneratorBenchmark();
10+
11+
@override
12+
void create(RunConfig config) {
13+
final workspace = config.workspace;
14+
final size = config.size;
15+
16+
// TODO(davidmorgan): add a way to pick `build` and generator versions.
17+
workspace.write(
18+
'pubspec.yaml',
19+
source: '''
20+
name: ${workspace.name}
21+
publish_to: none
22+
23+
environment:
24+
sdk: ^3.6.0
25+
26+
dependencies:
27+
json_annotation: any
28+
29+
dev_dependencies:
30+
build_runner: any
31+
json_serializable: any
32+
''',
33+
);
34+
35+
final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
36+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
37+
final libraryName = Benchmarks.libraryName(
38+
libraryNumber,
39+
benchmarkSize: size,
40+
);
41+
appLines.add("import '$libraryName';");
42+
}
43+
workspace.write(
44+
'lib/app.dart',
45+
source: appLines.map((l) => '$l\n').join(''),
46+
);
47+
48+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
49+
final libraryName = Benchmarks.libraryName(
50+
libraryNumber,
51+
benchmarkSize: size,
52+
);
53+
final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size);
54+
workspace.write(
55+
'lib/$libraryName',
56+
source: '''
57+
// ignore_for_file: unused_import
58+
import 'package:json_annotation/json_annotation.dart';
59+
60+
import 'app.dart';
61+
62+
part '$partName';
63+
64+
@JsonSerializable()
65+
class Value {
66+
Value();
67+
factory Value.fromJson(Map<String, dynamic> json) =>
68+
_\$ValueFromJson(json);
69+
Map<String, dynamic> toJson() => _\$ValueToJson(this);
70+
}
71+
''',
72+
);
73+
}
74+
}
75+
}

‎_benchmark/lib/benchmarks/mockito_generator_benchmark.dart

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import '../benchmark.dart';
6+
import '../config.dart';
7+
8+
class MockitoGeneratorBenchmark implements Benchmark {
9+
const MockitoGeneratorBenchmark();
10+
11+
@override
12+
void create(RunConfig config) {
13+
final workspace = config.workspace;
14+
final size = config.size;
15+
16+
// TODO(davidmorgan): add a way to pick `build` and generator versions.
17+
workspace.write(
18+
'pubspec.yaml',
19+
source: '''
20+
name: ${workspace.name}
21+
publish_to: none
22+
23+
environment:
24+
sdk: ^3.6.0
25+
26+
dependencies:
27+
mockito: any
28+
29+
dev_dependencies:
30+
build_runner: any
31+
''',
32+
);
33+
34+
final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
35+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
36+
final libraryName = Benchmarks.libraryName(
37+
libraryNumber,
38+
benchmarkSize: size,
39+
);
40+
appLines.add("import '$libraryName';");
41+
}
42+
workspace.write(
43+
'lib/app.dart',
44+
source: appLines.map((l) => '$l\n').join(''),
45+
);
46+
47+
for (var testNumber = 0; testNumber != size; ++testNumber) {
48+
final testLines = [
49+
'// ignore_for_file: unused import',
50+
'// CACHEBUSTER',
51+
"import 'package:mockito/annotations.dart';",
52+
];
53+
final libraryName = Benchmarks.libraryName(
54+
testNumber,
55+
benchmarkSize: size,
56+
);
57+
testLines.add("import 'package:${workspace.name}/$libraryName';");
58+
testLines.add('@GenerateNiceMocks([');
59+
testLines.add('MockSpec<Service$testNumber>(),');
60+
testLines.add('])');
61+
testLines.add("import 'some_test.mocks.dart';");
62+
63+
final testName = Benchmarks.testName(testNumber, benchmarkSize: size);
64+
workspace.write(
65+
'test/$testName',
66+
source: testLines.map((l) => '$l\n').join(''),
67+
);
68+
}
69+
70+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
71+
final libraryName = Benchmarks.libraryName(
72+
libraryNumber,
73+
benchmarkSize: size,
74+
);
75+
workspace.write(
76+
'lib/$libraryName',
77+
source: '''
78+
// ignore_for_file: unused_import
79+
import 'app.dart';
80+
81+
class Service$libraryNumber {
82+
void doSomething(int value) {}
83+
}
84+
''',
85+
);
86+
}
87+
}
88+
}

0 commit comments

Comments
 (0)