Skip to content

Commit 555dae1

Browse files
halter73halter73
committed
Add HTTP/2 support to PR benchmarks
1 parent a04bb1f commit 555dae1

File tree

4 files changed

+100
-18
lines changed

4 files changed

+100
-18
lines changed

‎src/JobConsumer/Program.cs

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ private static async Task<BenchmarkResult> BenchmarkPR(FileInfo processingFile,
147147
RunCommand($"git checkout {buildRules.BaselineSHA}");
148148
RunBuildCommands(buildRules);
149149

150-
var baselineArguments = GetDriverArguments(processingFile.FullName, session, sdkVersion, isBaseline: true);
150+
var baselineArguments = GetDriverArguments(processingFile.FullName, session, sdkVersion, buildRules, isBaseline: true);
151151

152152
outputBuilder.AppendLine($"Starting baseline run on '{buildRules.BaselineSHA}'...");
153153
var baselineSuccess = await RunDriver(baselineArguments, outputBuilder, errorBuilder);
@@ -172,7 +172,7 @@ private static async Task<BenchmarkResult> BenchmarkPR(FileInfo processingFile,
172172
RunCommand($"git checkout {session}");
173173
RunBuildCommands(buildRules);
174174

175-
var prArguments = GetDriverArguments(processingFile.FullName, session, sdkVersion, isBaseline: false);
175+
var prArguments = GetDriverArguments(processingFile.FullName, session, sdkVersion, buildRules, isBaseline: false);
176176

177177
outputBuilder.AppendLine($"Starting PR run on '{buildRules.PullRequestSHA}'...");
178178
var prSuccess = await RunDriver(prArguments, outputBuilder, errorBuilder);
@@ -434,17 +434,21 @@ private static async Task<string> GetSdkVersionOrNull()
434434
return null;
435435
}
436436

437-
private static string GetDriverArguments(string jobsFilePath, string sessionId, string sdkVersion, bool isBaseline)
437+
private static string GetDriverArguments(
438+
string jobsFilePath,
439+
string sessionId,
440+
string sdkVersion,
441+
BuildInstructions buildInstructions,
442+
bool isBaseline)
438443
{
439-
var argumentsBuilder = new StringBuilder($"{DriverPath} --server {ServerUrl} --client {ClientUrl} --jobs {jobsFilePath} --session {sessionId} --self-contained --aspNetCoreVersion Latest --runtimeVersion Latest --quiet");
444+
var argumentsBuilder = new StringBuilder($"{DriverPath} --server {ServerUrl} --client {ClientUrl} --jobs {jobsFilePath} --session {sessionId}");
440445

441-
if (isBaseline)
442-
{
443-
argumentsBuilder.Append(" --save baseline --description Before");
444-
}
445-
else
446+
argumentsBuilder.Append(" --self-contained --aspNetCoreVersion Latest --runtimeVersion Latest --quiet");
447+
448+
if (!string.IsNullOrWhiteSpace(buildInstructions.ScenarioName) && !string.Equals("Default", buildInstructions.ScenarioName, StringComparison.OrdinalIgnoreCase))
446449
{
447-
argumentsBuilder.Append(" --diff baseline --description After");
450+
argumentsBuilder.Append(" --scenario ");
451+
argumentsBuilder.Append(buildInstructions.ScenarioName);
448452
}
449453

450454
if (!string.IsNullOrWhiteSpace(sdkVersion))
@@ -453,6 +457,21 @@ private static string GetDriverArguments(string jobsFilePath, string sessionId,
453457
argumentsBuilder.Append(sdkVersion);
454458
}
455459

460+
if (!string.IsNullOrWhiteSpace(buildInstructions.ExtraDriverArgs))
461+
{
462+
argumentsBuilder.Append(" ");
463+
argumentsBuilder.Append(buildInstructions.ExtraDriverArgs);
464+
}
465+
466+
if (isBaseline)
467+
{
468+
argumentsBuilder.Append(" --save baseline --description Before");
469+
}
470+
else
471+
{
472+
argumentsBuilder.Append(" --diff baseline --description After");
473+
}
474+
456475
return argumentsBuilder.ToString();
457476
}
458477

@@ -464,6 +483,9 @@ private class BuildInstructions
464483
public int PullRequestNumber { get; set; }
465484
public string BaselineSHA { get; set; }
466485
public string PullRequestSHA { get; set; }
486+
487+
public string ScenarioName { get; set; }
488+
public string ExtraDriverArgs { get; set; }
467489
}
468490

469491
private class BenchmarkResult

‎src/PRJobProducer/BaseJobs/kestrel-pipelined-plaintext.json

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,28 @@
1515
},
1616
"ReadyStateText": "Application started.",
1717
"Path": "/plaintext",
18+
1819
"OutputFiles": [
1920
"artifacts/bin/Microsoft.AspNetCore.Server.Kestrel/Release/netcoreapp5.0/*.dll"
2021
],
2122
"DotNetTrace": false,
2223
"SelfContained": true
2324
},
25+
26+
"http2": {
27+
28+
"Client": "h2load",
29+
"ClientProperties": {
30+
"Streams": 70
31+
},
32+
33+
"PresetHeaders": "None",
34+
"Connections": 4,
35+
"Threads": 4,
36+
"SkipStartupLatencies": true,
37+
"ExtraDriverArgs": "--scheme h2c"
38+
},
39+
2440
"BuildInstructions": {
2541
"BuildCommands": [
2642
"./build.sh /t:Restore",

‎src/PRJobProducer/PRBenchmarkRequest.cs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using Octokit;
5+
6+
namespace PRJobProducer
7+
{
8+
public class PRBenchmarkRequest
9+
{
10+
public PullRequest PullRequest { get; set; }
11+
public string ScenarioName { get; set; }
12+
}
13+
}

‎src/PRJobProducer/Program.cs

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public class Program
2525
private const string ProcessedDirectoryName = "processed";
2626

2727
private const string BenchmarkRequest = "@aspnet-hello benchmark";
28-
private const string StartingBencmarkComment = "Starting pipelined plaintext benchmark with session ID '{0}'. This could take up to 30 minutes...";
28+
private const string StartingBencmarkComment = "Starting '{0}' pipelined plaintext benchmark with session ID '{1}'. This could take up to 30 minutes...";
2929
private const string CompletedBenchmarkCommentTemplate = "## Baseline\n\n```\n{0}\n```\n\n## PR\n\n```\n{1}\n```";
3030

3131
private static readonly DateTime CommentCutoffDate = DateTime.Now.AddHours(-24);
@@ -130,20 +130,22 @@ public static int Main(string[] args)
130130

131131
Console.WriteLine($"Scanning for benchmark requests in {Owner}/{Repo}.");
132132

133-
await foreach (var pr in GetPRsToBenchmark(client, botLoginName))
133+
await foreach (var prBenchmarkRequest in GetPRsToBenchmark(client, botLoginName))
134134
{
135+
var pr = prBenchmarkRequest.PullRequest;
136+
135137
try
136138
{
137139
var session = Guid.NewGuid().ToString("n");
138140
var newJobFileName = $"{session}.{Path.GetFileName(BaseJobPath)}";
139-
var startingCommentText = string.Format(StartingBencmarkComment, session);
141+
var startingCommentText = string.Format(StartingBencmarkComment, prBenchmarkRequest.ScenarioName, session);
140142

141-
Console.WriteLine($"Requesting benchmark for PR #{pr.Number}.");
143+
Console.WriteLine($"Requesting {prBenchmarkRequest.ScenarioName} benchmark for PR #{pr.Number}.");
142144
Console.WriteLine($"Benchmark starting comment: {startingCommentText}");
143145

144146
await client.Issue.Comment.Create(Owner, Repo, pr.Number, startingCommentText);
145147

146-
await RequestBenchmark(pr, newJobFileName);
148+
await RequestBenchmark(prBenchmarkRequest, newJobFileName);
147149

148150
Console.WriteLine($"Benchmark requested for PR #{pr.Number}. Waiting up to {BenchmarkTimeout} for results.");
149151
var results = await WaitForBenchmarkResults(newJobFileName);
@@ -179,7 +181,7 @@ string FormatOutput(string stdout, string stderr)
179181
return app.Execute(args);
180182
}
181183

182-
private static async IAsyncEnumerable<PullRequest> GetPRsToBenchmark(GitHubClient client, string botLoginName)
184+
private static async IAsyncEnumerable<PRBenchmarkRequest> GetPRsToBenchmark(GitHubClient client, string botLoginName)
183185
{
184186
var prRequest = new PullRequestRequest()
185187
{
@@ -210,7 +212,18 @@ private static async IAsyncEnumerable<PullRequest> GetPRsToBenchmark(GitHubClien
210212

211213
if (comment.Body.StartsWith(BenchmarkRequest) && await client.Organization.Member.CheckMember(Owner, comment.User.Login))
212214
{
213-
yield return pr;
215+
var scenarioName = comment.Body.Substring(BenchmarkRequest.Length).Trim();
216+
217+
if (string.IsNullOrWhiteSpace(scenarioName))
218+
{
219+
scenarioName = "Default";
220+
}
221+
222+
yield return new PRBenchmarkRequest
223+
{
224+
ScenarioName = scenarioName,
225+
PullRequest = pr,
226+
};
214227
}
215228
else if (comment.User.Login.Equals(botLoginName, StringComparison.OrdinalIgnoreCase))
216229
{
@@ -244,18 +257,33 @@ private static async Task<string[]> GetBuildCommands()
244257
return buildInstructions.BuildCommands;
245258
}
246259

247-
private static async Task RequestBenchmark(PullRequest pr, string newJobFileName)
260+
private static async Task RequestBenchmark(PRBenchmarkRequest prBenchmarkRequest, string newJobFileName)
248261
{
249262
await using var baseJobStream = File.OpenRead(BaseJobPath);
250263

251264
var jsonDictionary = await JsonSerializer.DeserializeAsync<Dictionary<string, object>>(baseJobStream);
252265

266+
var extraDriverArgs = "";
267+
if (jsonDictionary.ContainsKey(prBenchmarkRequest.ScenarioName))
268+
{
269+
var scenarioElement = (JsonElement)jsonDictionary[prBenchmarkRequest.ScenarioName];
270+
271+
if (scenarioElement.TryGetProperty("ExtraDriverArgs", out var extraDriverArgsElement))
272+
{
273+
extraDriverArgs = extraDriverArgsElement.GetString();
274+
}
275+
}
276+
277+
var pr = prBenchmarkRequest.PullRequest;
278+
253279
jsonDictionary["BuildInstructions"] = new BuildInstructions
254280
{
255281
BuildCommands = BuildCommands,
256282
PullRequestNumber = pr.Number,
257283
BaselineSHA = pr.Base.Sha,
258284
PullRequestSHA = pr.Head.Sha,
285+
ScenarioName = prBenchmarkRequest.ScenarioName,
286+
ExtraDriverArgs = extraDriverArgs,
259287
};
260288

261289
using var newJobStream = new MemoryStream();
@@ -410,6 +438,9 @@ private class BuildInstructions
410438
public int PullRequestNumber { get; set; }
411439
public string BaselineSHA { get; set; }
412440
public string PullRequestSHA { get; set; }
441+
442+
public string ScenarioName { get; set; }
443+
public string ExtraDriverArgs { get; set; }
413444
}
414445

415446
private class BenchmarkResult

0 commit comments

Comments
 (0)