Skip to content

Commit afd2645

Browse files
davidmorgandavidmorgan
authored
Add benchmark tool. (#3802)
1 parent 5ce397c commit afd2645

12 files changed

+732
-0
lines changed

‎_benchmark/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Benchmarks `build_runner` against synthetic codebases applying real generators:
2+
`built_value`, `freezed`, `json_serializable` or `mockito`.
3+
4+
Example usage:
5+
6+
```
7+
dart run _benchmark --generator=built_value benchmark
8+
```

‎_benchmark/bin/_benchmark.dart

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:io';
6+
7+
import 'package:_benchmark/commands.dart';
8+
import 'package:_benchmark/generators.dart';
9+
import 'package:args/command_runner.dart';
10+
11+
final commandRunner =
12+
CommandRunner<void>(
13+
'dart run _benchmark',
14+
'Benchmarks build_runner performance.',
15+
)
16+
..addCommand(BenchmarkCommand())
17+
..addCommand(MeasureCommand())
18+
..addCommand(CreateCommand())
19+
..argParser.addOption(
20+
'generator',
21+
help: 'Generator to benchmark.',
22+
allowed: Generator.values.map((e) => e.packageName).toList(),
23+
defaultsTo: Generator.builtValue.packageName,
24+
)
25+
..argParser.addOption(
26+
'root-directory',
27+
help: 'Root directory for generated source and builds.',
28+
defaultsTo: '${Directory.systemTemp.path}/build_benchmark',
29+
);
30+
31+
Future<void> main(List<String> arguments) async {
32+
await commandRunner.run(arguments);
33+
}

‎_benchmark/lib/benchmark.dart

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'config.dart';
6+
7+
/// A `build_runner` benchmark.
8+
abstract interface class Benchmark {
9+
void create(RunConfig config);
10+
}
11+
12+
/// Helpers for creating benchmarks.
13+
class Benchmarks {
14+
static String libraryName(int libraryNumber, {required int benchmarkSize}) {
15+
// Start numbering from 1.
16+
++libraryNumber;
17+
// Pad with zeros so alphabetic sort gives numerical ordering.
18+
final sizeDigits = benchmarkSize.toString().length;
19+
return 'lib${libraryNumber.toString().padLeft(sizeDigits, '0')}.dart';
20+
}
21+
22+
static String partName(
23+
int libraryNumber, {
24+
required int benchmarkSize,
25+
String infix = 'g',
26+
}) => libraryName(
27+
libraryNumber,
28+
benchmarkSize: benchmarkSize,
29+
).replaceAll('.dart', '.$infix.dart');
30+
31+
static String testName(int testNumber, {required int benchmarkSize}) {
32+
// Start numbering from 1.
33+
++testNumber;
34+
// Pad with zeros so alphabetic sort gives numerical ordering.
35+
final sizeDigits = benchmarkSize.toString().length;
36+
return 'some_test${testNumber.toString().padLeft(sizeDigits, '0')}.dart';
37+
}
38+
}

‎_benchmark/lib/benchmarks/built_value_generator_benchmark.dart

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import '../benchmark.dart';
6+
import '../config.dart';
7+
8+
/// Benchmark with one trivial `built_value` value type per library.
9+
///
10+
/// There is one large library cycle due to `app.dart` which depends on
11+
/// everything and is depended on by everything.
12+
class BuiltValueGeneratorBenchmark implements Benchmark {
13+
const BuiltValueGeneratorBenchmark();
14+
15+
@override
16+
void create(RunConfig config) {
17+
final workspace = config.workspace;
18+
final size = config.size;
19+
20+
// TODO(davidmorgan): add a way to pick `build` and generator versions.
21+
workspace.write(
22+
'pubspec.yaml',
23+
source: '''
24+
name: ${workspace.name}
25+
publish_to: none
26+
27+
environment:
28+
sdk: ^3.6.0
29+
30+
dependencies:
31+
built_value: any
32+
33+
dev_dependencies:
34+
build_runner: any
35+
built_value_generator: any
36+
''',
37+
);
38+
39+
final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
40+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
41+
final libraryName = Benchmarks.libraryName(
42+
libraryNumber,
43+
benchmarkSize: size,
44+
);
45+
appLines.add("import '$libraryName';");
46+
}
47+
workspace.write(
48+
'lib/app.dart',
49+
source: appLines.map((l) => '$l\n').join(''),
50+
);
51+
52+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
53+
final libraryName = Benchmarks.libraryName(
54+
libraryNumber,
55+
benchmarkSize: size,
56+
);
57+
final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size);
58+
workspace.write(
59+
'lib/$libraryName',
60+
source: '''
61+
// ignore_for_file: unused_import
62+
import 'package:built_value/built_value.dart';
63+
64+
import 'app.dart';
65+
66+
part '$partName';
67+
68+
abstract class Value implements Built<Value, ValueBuilder> {
69+
Value._();
70+
factory Value(void Function(ValueBuilder) updates) = _\$Value;
71+
}
72+
''',
73+
);
74+
}
75+
}
76+
}

‎_benchmark/lib/benchmarks/freezed_generator_benchmark.dart

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import '../benchmark.dart';
6+
import '../config.dart';
7+
8+
/// Benchmark with one trivial `freezed` value type per library.
9+
///
10+
/// There is one large library cycle due to `app.dart` which depends on
11+
/// everything and is depended on by everything.
12+
class FreezedGeneratorBenchmark implements Benchmark {
13+
const FreezedGeneratorBenchmark();
14+
15+
@override
16+
void create(RunConfig config) {
17+
final workspace = config.workspace;
18+
final size = config.size;
19+
20+
// TODO(davidmorgan): add a way to pick `build` and generator versions.
21+
workspace.write(
22+
'pubspec.yaml',
23+
source: '''
24+
name: ${workspace.name}
25+
publish_to: none
26+
27+
environment:
28+
sdk: ^3.6.0
29+
30+
dependencies:
31+
freezed_annotation: any
32+
33+
dev_dependencies:
34+
build_runner: any
35+
freezed: any
36+
''',
37+
);
38+
39+
final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
40+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
41+
final libraryName = Benchmarks.libraryName(
42+
libraryNumber,
43+
benchmarkSize: size,
44+
);
45+
appLines.add("import '$libraryName';");
46+
}
47+
workspace.write(
48+
'lib/app.dart',
49+
source: appLines.map((l) => '$l\n').join(''),
50+
);
51+
52+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
53+
final libraryName = Benchmarks.libraryName(
54+
libraryNumber,
55+
benchmarkSize: size,
56+
);
57+
final partName = Benchmarks.partName(
58+
libraryNumber,
59+
benchmarkSize: size,
60+
infix: 'freezed',
61+
);
62+
workspace.write(
63+
'lib/$libraryName',
64+
source: '''
65+
// ignore_for_file: unused_import
66+
import 'package:freezed_annotation/freezed_annotation.dart';
67+
68+
import 'app.dart';
69+
70+
part '$partName';
71+
72+
@freezed
73+
class Value with _\$Value {
74+
const factory Value() = _Value;
75+
}
76+
''',
77+
);
78+
}
79+
}
80+
}

‎_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import '../benchmark.dart';
6+
import '../config.dart';
7+
8+
/// Benchmark with one trivial `json_serializable` serializable type per
9+
/// library.
10+
///
11+
/// There is one large library cycle due to `app.dart` which depends on
12+
/// everything and is depended on by everything.
13+
class JsonSerializableGeneratorBenchmark implements Benchmark {
14+
const JsonSerializableGeneratorBenchmark();
15+
16+
@override
17+
void create(RunConfig config) {
18+
final workspace = config.workspace;
19+
final size = config.size;
20+
21+
// TODO(davidmorgan): add a way to pick `build` and generator versions.
22+
workspace.write(
23+
'pubspec.yaml',
24+
source: '''
25+
name: ${workspace.name}
26+
publish_to: none
27+
28+
environment:
29+
sdk: ^3.6.0
30+
31+
dependencies:
32+
json_annotation: any
33+
34+
dev_dependencies:
35+
build_runner: any
36+
json_serializable: any
37+
''',
38+
);
39+
40+
final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
41+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
42+
final libraryName = Benchmarks.libraryName(
43+
libraryNumber,
44+
benchmarkSize: size,
45+
);
46+
appLines.add("import '$libraryName';");
47+
}
48+
workspace.write(
49+
'lib/app.dart',
50+
source: appLines.map((l) => '$l\n').join(''),
51+
);
52+
53+
for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
54+
final libraryName = Benchmarks.libraryName(
55+
libraryNumber,
56+
benchmarkSize: size,
57+
);
58+
final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size);
59+
workspace.write(
60+
'lib/$libraryName',
61+
source: '''
62+
// ignore_for_file: unused_import
63+
import 'package:json_annotation/json_annotation.dart';
64+
65+
import 'app.dart';
66+
67+
part '$partName';
68+
69+
@JsonSerializable()
70+
class Value {
71+
Value();
72+
factory Value.fromJson(Map<String, dynamic> json) =>
73+
_\$ValueFromJson(json);
74+
Map<String, dynamic> toJson() => _\$ValueToJson(this);
75+
}
76+
''',
77+
);
78+
}
79+
}
80+
}

0 commit comments

Comments
 (0)