Skip to content

Commit 49f1827

Browse files
greeble-devgreeble-dev
authored
Speed up ECS benchmarks by limiting variations (#18659)
## Objective Reduce the time spent on ECS benchmarks without significantly compromising coverage. ## Background A `cargo bench -p benches --bench ecs` takes about 45 minutes. I'm guessing this bench is mainly used to check for regressions after ECS changes, and requiring 2x45 minute tests means that most people will skip benchmarking entirely. I noticed that some benches are repeated with sizes from long linear progressions (10, 20, ..., 100). This might be nice for detailed profiling, but seems too much for a overall regression check. ## Solution The PR follows the principles of "three or four different sizes is fine" and "powers of ten where it fits". The number of benches is reduced from 394 to 238 (-40%), and time from 46.2 minutes to 32.8 (-30%). While some coverage is lost, I think it's reasonable for anyone doing detailed profiling of a particular feature to temporarily add more benches. There's a couple of changes to avoid leading zeroes. I felt that `0010, 0100, 1000` is harder to read than `10, 100, 1000`. ## Is That Enough? 32 minutes is still too much. Possible future options: - Reduce measurement and warmup times. I suspect the current times (mostly 4-5 seconds total) are too conservative, and 1 second would be fine for spotting significant regressions. - Split the bench into quick and detailed variants. ## Testing ``` cargo bench -p benches --bench ecs ```
1 parent 023b502 commit 49f1827

File tree

9 files changed

+42
-51
lines changed

9 files changed

+42
-51
lines changed

‎benches/benches/bevy_ecs/components/archetype_updates.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,7 @@ fn add_archetypes(world: &mut World, count: u16) {
5151

5252
pub fn no_archetypes(criterion: &mut Criterion) {
5353
let mut group = criterion.benchmark_group("no_archetypes");
54-
for i in 0..=5 {
55-
let system_count = i * 20;
54+
for system_count in [0, 10, 100] {
5655
let (mut world, mut schedule) = setup(system_count);
5756
group.bench_with_input(
5857
BenchmarkId::new("system_count", system_count),
@@ -69,7 +68,7 @@ pub fn no_archetypes(criterion: &mut Criterion) {
6968
pub fn added_archetypes(criterion: &mut Criterion) {
7069
const SYSTEM_COUNT: usize = 100;
7170
let mut group = criterion.benchmark_group("added_archetypes");
72-
for archetype_count in [100, 200, 500, 1000, 2000, 5000, 10000] {
71+
for archetype_count in [100, 1_000, 10_000] {
7372
group.bench_with_input(
7473
BenchmarkId::new("archetype_count", archetype_count),
7574
&archetype_count,

‎benches/benches/bevy_ecs/empty_archetypes.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ fn add_archetypes(world: &mut World, count: u16) {
155155

156156
fn empty_archetypes(criterion: &mut Criterion) {
157157
let mut group = criterion.benchmark_group("empty_archetypes");
158-
for archetype_count in [10, 100, 500, 1000, 2000, 5000, 10000] {
158+
for archetype_count in [10, 100, 1_000, 10_000] {
159159
let (mut world, mut schedule) = setup(true, |schedule| {
160160
schedule.add_systems(iter);
161161
});
@@ -186,7 +186,7 @@ fn empty_archetypes(criterion: &mut Criterion) {
186186
},
187187
);
188188
}
189-
for archetype_count in [10, 100, 500, 1000, 2000, 5000, 10000] {
189+
for archetype_count in [10, 100, 1_000, 10_000] {
190190
let (mut world, mut schedule) = setup(true, |schedule| {
191191
schedule.add_systems(for_each);
192192
});
@@ -217,7 +217,7 @@ fn empty_archetypes(criterion: &mut Criterion) {
217217
},
218218
);
219219
}
220-
for archetype_count in [10, 100, 500, 1000, 2000, 5000, 10000] {
220+
for archetype_count in [10, 100, 1_000, 10_000] {
221221
let (mut world, mut schedule) = setup(true, |schedule| {
222222
schedule.add_systems(par_for_each);
223223
});

‎benches/benches/bevy_ecs/events/mod.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,19 @@ fn send(c: &mut Criterion) {
99
let mut group = c.benchmark_group("events_send");
1010
group.warm_up_time(core::time::Duration::from_millis(500));
1111
group.measurement_time(core::time::Duration::from_secs(4));
12-
for count in [100, 1000, 10000, 50000] {
12+
for count in [100, 1_000, 10_000] {
1313
group.bench_function(format!("size_4_events_{}", count), |b| {
1414
let mut bench = send::Benchmark::<4>::new(count);
1515
b.iter(move || bench.run());
1616
});
1717
}
18-
for count in [100, 1000, 10000, 50000] {
18+
for count in [100, 1_000, 10_000] {
1919
group.bench_function(format!("size_16_events_{}", count), |b| {
2020
let mut bench = send::Benchmark::<16>::new(count);
2121
b.iter(move || bench.run());
2222
});
2323
}
24-
for count in [100, 1000, 10000, 50000] {
24+
for count in [100, 1_000, 10_000] {
2525
group.bench_function(format!("size_512_events_{}", count), |b| {
2626
let mut bench = send::Benchmark::<512>::new(count);
2727
b.iter(move || bench.run());
@@ -34,19 +34,19 @@ fn iter(c: &mut Criterion) {
3434
let mut group = c.benchmark_group("events_iter");
3535
group.warm_up_time(core::time::Duration::from_millis(500));
3636
group.measurement_time(core::time::Duration::from_secs(4));
37-
for count in [100, 1000, 10000, 50000] {
37+
for count in [100, 1_000, 10_000] {
3838
group.bench_function(format!("size_4_events_{}", count), |b| {
3939
let mut bench = iter::Benchmark::<4>::new(count);
4040
b.iter(move || bench.run());
4141
});
4242
}
43-
for count in [100, 1000, 10000, 50000] {
43+
for count in [100, 1_000, 10_000] {
4444
group.bench_function(format!("size_16_events_{}", count), |b| {
4545
let mut bench = iter::Benchmark::<4>::new(count);
4646
b.iter(move || bench.run());
4747
});
4848
}
49-
for count in [100, 1000, 10000, 50000] {
49+
for count in [100, 1_000, 10_000] {
5050
group.bench_function(format!("size_512_events_{}", count), |b| {
5151
let mut bench = iter::Benchmark::<512>::new(count);
5252
b.iter(move || bench.run());

‎benches/benches/bevy_ecs/scheduling/run_condition.rs‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,14 @@ pub fn run_condition_yes(criterion: &mut Criterion) {
1717
group.warm_up_time(core::time::Duration::from_millis(500));
1818
group.measurement_time(core::time::Duration::from_secs(3));
1919
fn empty() {}
20-
for amount in 0..21 {
20+
for amount in [10, 100, 1_000] {
2121
let mut schedule = Schedule::default();
22-
schedule.add_systems(empty.run_if(yes));
23-
for _ in 0..amount {
22+
for _ in 0..(amount / 5) {
2423
schedule.add_systems((empty, empty, empty, empty, empty).distributive_run_if(yes));
2524
}
2625
// run once to initialize systems
2726
schedule.run(&mut world);
28-
group.bench_function(format!("{:03}_systems", 5 * amount + 1), |bencher| {
27+
group.bench_function(format!("{}_systems", amount), |bencher| {
2928
bencher.iter(|| {
3029
schedule.run(&mut world);
3130
});
@@ -40,15 +39,14 @@ pub fn run_condition_no(criterion: &mut Criterion) {
4039
group.warm_up_time(core::time::Duration::from_millis(500));
4140
group.measurement_time(core::time::Duration::from_secs(3));
4241
fn empty() {}
43-
for amount in 0..21 {
42+
for amount in [10, 100, 1_000] {
4443
let mut schedule = Schedule::default();
45-
schedule.add_systems(empty.run_if(no));
46-
for _ in 0..amount {
44+
for _ in 0..(amount / 5) {
4745
schedule.add_systems((empty, empty, empty, empty, empty).distributive_run_if(no));
4846
}
4947
// run once to initialize systems
5048
schedule.run(&mut world);
51-
group.bench_function(format!("{:03}_systems", 5 * amount + 1), |bencher| {
49+
group.bench_function(format!("{}_systems", amount), |bencher| {
5250
bencher.iter(|| {
5351
schedule.run(&mut world);
5452
});
@@ -70,17 +68,16 @@ pub fn run_condition_yes_with_query(criterion: &mut Criterion) {
7068
fn yes_with_query(query: Single<&TestBool>) -> bool {
7169
query.0
7270
}
73-
for amount in 0..21 {
71+
for amount in [10, 100, 1_000] {
7472
let mut schedule = Schedule::default();
75-
schedule.add_systems(empty.run_if(yes_with_query));
76-
for _ in 0..amount {
73+
for _ in 0..(amount / 5) {
7774
schedule.add_systems(
7875
(empty, empty, empty, empty, empty).distributive_run_if(yes_with_query),
7976
);
8077
}
8178
// run once to initialize systems
8279
schedule.run(&mut world);
83-
group.bench_function(format!("{:03}_systems", 5 * amount + 1), |bencher| {
80+
group.bench_function(format!("{}_systems", amount), |bencher| {
8481
bencher.iter(|| {
8582
schedule.run(&mut world);
8683
});
@@ -99,17 +96,16 @@ pub fn run_condition_yes_with_resource(criterion: &mut Criterion) {
9996
fn yes_with_resource(res: Res<TestBool>) -> bool {
10097
res.0
10198
}
102-
for amount in 0..21 {
99+
for amount in [10, 100, 1_000] {
103100
let mut schedule = Schedule::default();
104-
schedule.add_systems(empty.run_if(yes_with_resource));
105-
for _ in 0..amount {
101+
for _ in 0..(amount / 5) {
106102
schedule.add_systems(
107103
(empty, empty, empty, empty, empty).distributive_run_if(yes_with_resource),
108104
);
109105
}
110106
// run once to initialize systems
111107
schedule.run(&mut world);
112-
group.bench_function(format!("{:03}_systems", 5 * amount + 1), |bencher| {
108+
group.bench_function(format!("{}_systems", amount), |bencher| {
113109
bencher.iter(|| {
114110
schedule.run(&mut world);
115111
});

‎benches/benches/bevy_ecs/scheduling/running_systems.rs‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,25 +20,25 @@ pub fn empty_systems(criterion: &mut Criterion) {
2020
group.warm_up_time(core::time::Duration::from_millis(500));
2121
group.measurement_time(core::time::Duration::from_secs(3));
2222
fn empty() {}
23-
for amount in 0..5 {
23+
for amount in [0, 2, 4] {
2424
let mut schedule = Schedule::default();
2525
for _ in 0..amount {
2626
schedule.add_systems(empty);
2727
}
2828
schedule.run(&mut world);
29-
group.bench_function(format!("{:03}_systems", amount), |bencher| {
29+
group.bench_function(format!("{}_systems", amount), |bencher| {
3030
bencher.iter(|| {
3131
schedule.run(&mut world);
3232
});
3333
});
3434
}
35-
for amount in 1..21 {
35+
for amount in [10, 100, 1_000] {
3636
let mut schedule = Schedule::default();
37-
for _ in 0..amount {
37+
for _ in 0..(amount / 5) {
3838
schedule.add_systems((empty, empty, empty, empty, empty));
3939
}
4040
schedule.run(&mut world);
41-
group.bench_function(format!("{:03}_systems", 5 * amount), |bencher| {
41+
group.bench_function(format!("{}_systems", amount), |bencher| {
4242
bencher.iter(|| {
4343
schedule.run(&mut world);
4444
});
@@ -67,23 +67,21 @@ pub fn busy_systems(criterion: &mut Criterion) {
6767
let mut group = criterion.benchmark_group("busy_systems");
6868
group.warm_up_time(core::time::Duration::from_millis(500));
6969
group.measurement_time(core::time::Duration::from_secs(3));
70-
for entity_bunches in 1..6 {
70+
for entity_bunches in [1, 3, 5] {
7171
world.spawn_batch((0..4 * ENTITY_BUNCH).map(|_| (A(0.0), B(0.0))));
7272
world.spawn_batch((0..4 * ENTITY_BUNCH).map(|_| (A(0.0), B(0.0), C(0.0))));
7373
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (A(0.0), B(0.0), C(0.0), D(0.0))));
7474
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (A(0.0), B(0.0), C(0.0), E(0.0))));
75-
for system_amount in 0..5 {
75+
for system_amount in [3, 9, 15] {
7676
let mut schedule = Schedule::default();
77-
schedule.add_systems((ab, cd, ce));
78-
for _ in 0..system_amount {
77+
for _ in 0..(system_amount / 3) {
7978
schedule.add_systems((ab, cd, ce));
8079
}
8180
schedule.run(&mut world);
8281
group.bench_function(
8382
format!(
8483
"{:02}x_entities_{:02}_systems",
85-
entity_bunches,
86-
3 * system_amount + 3
84+
entity_bunches, system_amount
8785
),
8886
|bencher| {
8987
bencher.iter(|| {
@@ -119,22 +117,20 @@ pub fn contrived(criterion: &mut Criterion) {
119117
let mut group = criterion.benchmark_group("contrived");
120118
group.warm_up_time(core::time::Duration::from_millis(500));
121119
group.measurement_time(core::time::Duration::from_secs(3));
122-
for entity_bunches in 1..6 {
120+
for entity_bunches in [1, 3, 5] {
123121
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (A(0.0), B(0.0), C(0.0), D(0.0))));
124122
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (A(0.0), B(0.0))));
125123
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (C(0.0), D(0.0))));
126-
for system_amount in 0..5 {
124+
for system_amount in [3, 9, 15] {
127125
let mut schedule = Schedule::default();
128-
schedule.add_systems((s_0, s_1, s_2));
129-
for _ in 0..system_amount {
126+
for _ in 0..(system_amount / 3) {
130127
schedule.add_systems((s_0, s_1, s_2));
131128
}
132129
schedule.run(&mut world);
133130
group.bench_function(
134131
format!(
135132
"{:02}x_entities_{:02}_systems",
136-
entity_bunches,
137-
3 * system_amount + 3
133+
entity_bunches, system_amount
138134
),
139135
|bencher| {
140136
bencher.iter(|| {

‎benches/benches/bevy_ecs/world/commands.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ pub fn spawn_commands(criterion: &mut Criterion) {
3636
group.warm_up_time(core::time::Duration::from_millis(500));
3737
group.measurement_time(core::time::Duration::from_secs(4));
3838

39-
for entity_count in (1..5).map(|i| i * 2 * 1000) {
39+
for entity_count in [100, 1_000, 10_000] {
4040
group.bench_function(format!("{}_entities", entity_count), |bencher| {
4141
let mut world = World::default();
4242
let mut command_queue = CommandQueue::default();
@@ -136,7 +136,7 @@ pub fn fake_commands(criterion: &mut Criterion) {
136136
group.warm_up_time(core::time::Duration::from_millis(500));
137137
group.measurement_time(core::time::Duration::from_secs(4));
138138

139-
for command_count in (1..5).map(|i| i * 2 * 1000) {
139+
for command_count in [100, 1_000, 10_000] {
140140
group.bench_function(format!("{}_commands", command_count), |bencher| {
141141
let mut world = World::default();
142142
let mut command_queue = CommandQueue::default();
@@ -181,7 +181,7 @@ pub fn sized_commands_impl<T: Default + Command>(criterion: &mut Criterion) {
181181
group.warm_up_time(core::time::Duration::from_millis(500));
182182
group.measurement_time(core::time::Duration::from_secs(4));
183183

184-
for command_count in (1..5).map(|i| i * 2 * 1000) {
184+
for command_count in [100, 1_000, 10_000] {
185185
group.bench_function(format!("{}_commands", command_count), |bencher| {
186186
let mut world = World::default();
187187
let mut command_queue = CommandQueue::default();

‎benches/benches/bevy_ecs/world/despawn.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ pub fn world_despawn(criterion: &mut Criterion) {
1212
group.warm_up_time(core::time::Duration::from_millis(500));
1313
group.measurement_time(core::time::Duration::from_secs(4));
1414

15-
for entity_count in (0..5).map(|i| 10_u32.pow(i)) {
15+
for entity_count in [1, 100, 10_000] {
1616
group.bench_function(format!("{}_entities", entity_count), |bencher| {
1717
bencher.iter_batched_ref(
1818
|| {

‎benches/benches/bevy_ecs/world/despawn_recursive.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ pub fn world_despawn_recursive(criterion: &mut Criterion) {
1212
group.warm_up_time(core::time::Duration::from_millis(500));
1313
group.measurement_time(core::time::Duration::from_secs(4));
1414

15-
for entity_count in (0..5).map(|i| 10_u32.pow(i)) {
15+
for entity_count in [1, 100, 10_000] {
1616
group.bench_function(format!("{}_entities", entity_count), |bencher| {
1717
bencher.iter_batched_ref(
1818
|| {

‎benches/benches/bevy_ecs/world/spawn.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ pub fn world_spawn(criterion: &mut Criterion) {
1212
group.warm_up_time(core::time::Duration::from_millis(500));
1313
group.measurement_time(core::time::Duration::from_secs(4));
1414

15-
for entity_count in (0..5).map(|i| 10_u32.pow(i)) {
15+
for entity_count in [1, 100, 10_000] {
1616
group.bench_function(format!("{}_entities", entity_count), |bencher| {
1717
let mut world = World::default();
1818
bencher.iter(|| {

0 commit comments

Comments
 (0)