Skip to content

Commit b14e2f5

Browse files
caaavik-msftcaaavik-msft
authored
[PERF] Move performance testing logic into python scripts (#101143)
* Initial debugging work for python pipeline * Use custom performance branch * Remove template reference * Move resources to perf.yml * Ensure runtime repo is checked out to SourcesDirectory * Try fix python run * Set python in perf-job.yml * More fixes * More fixes * Set most params in perf-job.yml * Fix variable declaration * Fix some bugs in setup parameters * Ensure Helix Access Token is passed in * Fix osGroup arg parsing * Fix ios args and use local commit time * Fix physical promotion yaml * Fix project file path * Ensure internal param is passed * Ensure project file is passed in * Try dynamically load performance repo * Try use condition inside repositories definition * Try put condition on repositories object * Try use variables to conditionally create repository * Fix conditional variable definition * Just use ADO for performance repo * Use updated repository name for scenarios jobs * Use HELIX_WORKITEM_ROOT for performance dir * Fix paths for android APKs * Remove accidental submodule * Use new project file location for arm64 jit job * Move archive of work item dir to python * Use github for performance repo checkout on public jobs * Another attempt at conditional repository definition * Another attempt at conditional repository * Use multi repo checkout only for internal runs * Ensure ios scenarios and wasm jobs are working * Only copy files from dotnet-runtime directory on internal jobs * Use branch instead of ref for performance repository clone * Clone performance repo to correct folder in public jobs
1 parent 657f18d commit b14e2f5

19 files changed

+207
-1357
lines changed

‎eng/pipelines/common/perf-variables.yml‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎eng/pipelines/common/variables.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,4 @@ variables:
5858
ne(variables['isExtraPlatformsBuild'], true),
5959
eq(variables['isRollingBuild'], true))) ]
6060

61-
- template: /eng/pipelines/common/perf-variables.yml
62-
63-
- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml
61+
- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml

‎eng/pipelines/coreclr/perf-non-wasm-jobs.yml‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
parameters:
2+
perfBranch: 'main'
3+
14
jobs:
25

36
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}:
@@ -100,6 +103,7 @@ jobs:
100103
- template: /eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml
101104
parameters:
102105
hybridGlobalization: True
106+
perfBranch: ${{ parameters.perfBranch }}
103107

104108
# run android scenarios
105109
- template: /eng/pipelines/common/platform-matrix.yml
@@ -112,10 +116,11 @@ jobs:
112116
jobParameters:
113117
testGroup: perf
114118
runtimeType: AndroidMono
115-
projectFile: android_scenarios.proj
119+
projectFile: $(Build.SourcesDirectory)/eng/testing/performance/android_scenarios.proj
116120
runKind: android_scenarios
117121
runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
118122
logicalmachine: 'perfpixel4a'
123+
perfBranch: ${{ parameters.perfBranch }}
119124

120125
# run mono microbenchmarks perf job
121126
- template: /eng/pipelines/common/platform-matrix.yml
@@ -129,10 +134,11 @@ jobs:
129134
testGroup: perf
130135
liveLibrariesBuildConfig: Release
131136
runtimeType: mono
132-
projectFile: microbenchmarks.proj
137+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
133138
runKind: micro_mono
134139
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
135140
logicalmachine: 'perftiger'
141+
perfBranch: ${{ parameters.perfBranch }}
136142

137143
# run mono interpreter perf job
138144
- template: /eng/pipelines/common/platform-matrix.yml
@@ -147,10 +153,11 @@ jobs:
147153
liveLibrariesBuildConfig: Release
148154
runtimeType: mono
149155
codeGenType: 'Interpreter'
150-
projectFile: microbenchmarks.proj
156+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
151157
runKind: micro_mono
152158
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
153159
logicalmachine: 'perftiger'
160+
perfBranch: ${{ parameters.perfBranch }}
154161

155162
# run mono aot microbenchmarks perf job
156163
- template: /eng/pipelines/common/platform-matrix.yml
@@ -165,10 +172,11 @@ jobs:
165172
liveLibrariesBuildConfig: Release
166173
runtimeType: mono
167174
codeGenType: 'AOT'
168-
projectFile: microbenchmarks.proj
175+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
169176
runKind: micro_mono
170177
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
171178
logicalmachine: 'perftiger'
179+
perfBranch: ${{ parameters.perfBranch }}
172180

173181
# run coreclr perftiger microbenchmarks perf job
174182
- template: /eng/pipelines/common/platform-matrix.yml
@@ -184,10 +192,11 @@ jobs:
184192
jobParameters:
185193
testGroup: perf
186194
liveLibrariesBuildConfig: Release
187-
projectFile: microbenchmarks.proj
195+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
188196
runKind: micro
189197
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
190198
logicalmachine: 'perftiger'
199+
perfBranch: ${{ parameters.perfBranch }}
191200

192201
# run coreclr perftiger microbenchmarks no dynamic pgo perf jobs
193202
- template: /eng/pipelines/common/platform-matrix.yml
@@ -200,11 +209,12 @@ jobs:
200209
jobParameters:
201210
testGroup: perf
202211
liveLibrariesBuildConfig: Release
203-
projectFile: microbenchmarks.proj
212+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
204213
runKind: micro
205214
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
206215
logicalmachine: 'perftiger'
207216
pgoRunType: -NoDynamicPGO
217+
perfBranch: ${{ parameters.perfBranch }}
208218

209219
- template: /eng/pipelines/common/platform-matrix.yml
210220
parameters:
@@ -216,11 +226,12 @@ jobs:
216226
jobParameters:
217227
testGroup: perf
218228
liveLibrariesBuildConfig: Release
219-
projectFile: microbenchmarks.proj
229+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
220230
runKind: micro
221231
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
222232
logicalmachine: 'perftiger'
223233
pgoRunType: --nodynamicpgo
234+
perfBranch: ${{ parameters.perfBranch }}
224235

225236
# run coreclr perftiger microbenchmarks no R2R perf jobs
226237
- template: /eng/pipelines/common/platform-matrix.yml
@@ -233,11 +244,12 @@ jobs:
233244
jobParameters:
234245
testGroup: perf
235246
liveLibrariesBuildConfig: Release
236-
projectFile: microbenchmarks.proj
247+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
237248
runKind: micro
238249
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
239250
logicalmachine: 'perftiger'
240251
r2rRunType: -NoR2R
252+
perfBranch: ${{ parameters.perfBranch }}
241253

242254
- template: /eng/pipelines/common/platform-matrix.yml
243255
parameters:
@@ -249,11 +261,12 @@ jobs:
249261
jobParameters:
250262
testGroup: perf
251263
liveLibrariesBuildConfig: Release
252-
projectFile: microbenchmarks.proj
264+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
253265
runKind: micro
254266
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
255267
logicalmachine: 'perftiger'
256268
r2rRunType: --nor2r
269+
perfBranch: ${{ parameters.perfBranch }}
257270

258271
# run coreclr perfowl microbenchmarks perf job
259272
- template: /eng/pipelines/common/platform-matrix.yml
@@ -267,10 +280,11 @@ jobs:
267280
jobParameters:
268281
testGroup: perf
269282
liveLibrariesBuildConfig: Release
270-
projectFile: microbenchmarks.proj
283+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
271284
runKind: micro
272285
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
273286
logicalmachine: 'perfowl'
287+
perfBranch: ${{ parameters.perfBranch }}
274288

275289
# run coreclr perfviper microbenchmarks perf job
276290
- template: /eng/pipelines/common/platform-matrix.yml
@@ -284,10 +298,11 @@ jobs:
284298
jobParameters:
285299
testGroup: perf
286300
liveLibrariesBuildConfig: Release
287-
projectFile: microbenchmarks.proj
301+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
288302
runKind: micro
289303
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
290304
logicalmachine: 'perfviper'
305+
perfBranch: ${{ parameters.perfBranch }}
291306

292307
# run coreclr perfviper microbenchmarks perf jitoptrepeat jobs
293308
- template: /eng/pipelines/common/platform-matrix.yml
@@ -301,11 +316,12 @@ jobs:
301316
jobParameters:
302317
testGroup: perf
303318
liveLibrariesBuildConfig: Release
304-
projectFile: microbenchmarks.proj
319+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
305320
runKind: micro
306321
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
307322
logicalmachine: 'perfviper'
308323
experimentName: 'jitoptrepeat'
324+
perfBranch: ${{ parameters.perfBranch }}
309325

310326
# run coreclr crossgen perf job
311327
- template: /eng/pipelines/common/platform-matrix.yml
@@ -319,10 +335,11 @@ jobs:
319335
jobParameters:
320336
testGroup: perf
321337
liveLibrariesBuildConfig: Release
322-
projectFile: crossgen_perf.proj
338+
projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj
323339
runKind: crossgen_scenarios
324340
runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
325341
logicalmachine: 'perftiger_crossgen'
342+
perfBranch: ${{ parameters.perfBranch }}
326343

327344
# build mono runtime packs
328345
- template: /eng/pipelines/common/platform-matrix.yml
@@ -368,3 +385,4 @@ jobs:
368385
archiveExtension: '.tar.gz'
369386
archiveType: tar
370387
tarCompression: gz
388+
perfBranch: ${{ parameters.perfBranch }}

‎eng/pipelines/coreclr/perf-wasm-jobs.yml‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ parameters:
77
onlySanityCheck: false
88
downloadSpecificBuild: null # { buildId, pipeline, branchName, project }
99
collectHelixLogsScript: ''
10-
perfForkToUse: {} # url, branch
10+
perfBranch: 'main'
1111

1212
jobs:
1313

@@ -43,14 +43,14 @@ jobs:
4343
liveLibrariesBuildConfig: Release
4444
runtimeType: wasm
4545
codeGenType: 'wasm'
46-
projectFile: microbenchmarks.proj
46+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
4747
runKind: micro
4848
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
4949
logicalmachine: 'perftiger'
5050
javascriptEngine: 'javascriptcore'
5151
collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }}
5252
downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
53-
perfForkToUse: ${{ parameters.perfForkToUse }}
53+
perfBranch: ${{ parameters.perfBranch }}
5454
#downloadSpecificBuild:
5555
#buildId: '1693181'
5656
#pipeline: 'perf-wasm'
@@ -69,14 +69,14 @@ jobs:
6969
livelibrariesbuildconfig: Release
7070
runtimetype: wasm
7171
codegentype: 'aot'
72-
projectfile: microbenchmarks.proj
72+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
7373
runkind: micro
7474
runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
7575
logicalmachine: 'perftiger'
7676
javascriptengine: 'javascriptcore'
7777
collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }}
7878
downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
79-
perfForkToUse: ${{ parameters.perfForkToUse }}
79+
perfBranch: ${{ parameters.perfBranch }}
8080

8181
- ${{ if eq(parameters.runProfile, 'v8') }}:
8282
- ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}:
@@ -110,7 +110,7 @@ jobs:
110110
liveLibrariesBuildConfig: Release
111111
runtimeType: wasm
112112
codeGenType: 'wasm'
113-
projectFile: microbenchmarks.proj
113+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
114114
runKind: micro
115115
runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
116116
logicalmachine: 'perftiger'
@@ -121,7 +121,7 @@ jobs:
121121
compare: ${{ parameters.compare }}
122122
onlySanityCheck: ${{ parameters.onlySanityCheck }}
123123
downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
124-
perfForkToUse: ${{ parameters.perfForkToUse }}
124+
perfBranch: ${{ parameters.perfBranch }}
125125

126126
#run mono wasm aot microbenchmarks perf job
127127
# Disabled for runtime-wasm-perf on PRs due to https://github.com/dotnet/runtime/issues/95101
@@ -138,7 +138,7 @@ jobs:
138138
livelibrariesbuildconfig: Release
139139
runtimetype: wasm
140140
codegentype: 'aot'
141-
projectfile: microbenchmarks.proj
141+
projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
142142
runkind: micro
143143
runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
144144
logicalmachine: 'perftiger'
@@ -149,7 +149,7 @@ jobs:
149149
compare: ${{ parameters.compare }}
150150
onlySanityCheck: ${{ parameters.onlySanityCheck }}
151151
downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
152-
perfForkToUse: ${{ parameters.perfForkToUse }}
152+
perfBranch: ${{ parameters.perfBranch }}
153153

154154
# run mono wasm blazor perf job
155155
- template: /eng/pipelines/common/platform-matrix.yml
@@ -163,14 +163,14 @@ jobs:
163163
testGroup: perf
164164
liveLibrariesBuildConfig: Release
165165
runtimeType: wasm
166-
projectFile: blazor_perf.proj
166+
projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj
167167
runKind: blazor_scenarios
168168
runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
169169
# For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm)
170-
#additionalSetupParameters: '--dotnetversions 8.0.0' # passed to performance-setup.sh
170+
#additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py
171171
logicalmachine: 'perftiger'
172172
downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
173-
perfForkToUse: ${{ parameters.perfForkToUse }}
173+
perfBranch: ${{ parameters.perfBranch }}
174174

175175
- ${{if or(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}:
176176
# run mono wasm blazor perf job
@@ -185,12 +185,12 @@ jobs:
185185
testGroup: perf
186186
liveLibrariesBuildConfig: Release
187187
runtimeType: wasm
188-
projectFile: blazor_perf.proj
188+
projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj
189189
runKind: blazor_scenarios
190190
runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
191191
# For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm)
192-
#additionalSetupParameters: '--dotnetversions 8.0.0' # passed to performance-setup.sh
192+
#additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py
193193
logicalmachine: 'perftiger'
194194
downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
195-
perfForkToUse: ${{ parameters.perfForkToUse }}
196195
hybridGlobalization: True
196+
perfBranch: ${{ parameters.perfBranch }}

‎eng/pipelines/coreclr/perf.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
parameters:
2+
- name: perfBranch
3+
displayName: Performance Repo Branch
4+
type: string
5+
default: 'main'
6+
17
trigger:
28
batch: true
39
branches:
@@ -19,7 +25,7 @@ trigger:
1925
- THIRD-PARTY-NOTICES.TXT
2026

2127
variables:
22-
- template: /eng/pipelines/common/variables.yml
28+
- template: /eng/pipelines/common/variables.yml
2329

2430
#
2531
# For the 'schedule' case, only wasm/jsc perf jobs are run.
@@ -46,9 +52,12 @@ extends:
4652
- template: /eng/pipelines/coreclr/perf-wasm-jobs.yml
4753
parameters:
4854
collectHelixLogsScript: ${{ variables._wasmCollectHelixLogsScript }}
55+
perfBranch: ${{ parameters.perfBranch }}
4956
#${{ and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}:
5057
# runProfile: 'non-v8'
5158
${{ if ne(variables['System.TeamProject'], 'public') }}:
5259
runProfile: 'v8'
5360

5461
- template: /eng/pipelines/coreclr/perf-non-wasm-jobs.yml
62+
parameters:
63+
perfBranch: ${{ parameters.perfBranch }}

0 commit comments

Comments
 (0)