Throw when unsupported HTTPS options are set for HTTP/3

src/Servers/Kestrel/Core/src/ListenOptionsHttpsExtensions.cs

@@ -256,6 +256,10 @@ public static ListenOptions UseHttps(this ListenOptions listenOptions, ServerOpt
         /// <returns>The <see cref="ListenOptions"/>.</returns>
         public static ListenOptions UseHttps(this ListenOptions listenOptions, ServerOptionsSelectionCallback serverOptionsSelectionCallback, object state, TimeSpan handshakeTimeout)
         {
+            if (listenOptions.Protocols.HasFlag(HttpProtocols.Http3))
+            {
+                throw new NotSupportedException($"{nameof(UseHttps)} with {nameof(ServerOptionsSelectionCallback)} is not supported with HTTP/3.");
+            }
             return listenOptions.UseHttps(new TlsHandshakeCallbackOptions()
             {
                 OnConnection = context => serverOptionsSelectionCallback(context.SslStream, context.ClientHelloInfo, context.State, context.CancellationToken),
@@ -283,6 +287,11 @@ public static ListenOptions UseHttps(this ListenOptions listenOptions, TlsHandsh
                 throw new ArgumentException($"{nameof(TlsHandshakeCallbackOptions.OnConnection)} must not be null.");
             }
 
+            if (listenOptions.Protocols.HasFlag(HttpProtocols.Http3))
+            {
+                throw new NotSupportedException($"{nameof(UseHttps)} with {nameof(TlsHandshakeCallbackOptions)} is not supported with HTTP/3.");
+            }
+
             var loggerFactory = listenOptions.KestrelServerOptions?.ApplicationServices.GetRequiredService<ILoggerFactory>() ?? NullLoggerFactory.Instance;
 
             listenOptions.IsTls = true;

src/Servers/Kestrel/Core/src/Middleware/HttpsConnectionMiddleware.cs

@@ -510,6 +510,11 @@ private static bool IsWindowsVersionIncompatibleWithHttp2()
 
         internal static SslServerAuthenticationOptions CreateHttp3Options(HttpsConnectionAdapterOptions httpsOptions)
         {
+            if (httpsOptions.OnAuthenticate != null)
+            {
+                throw new NotSupportedException($"The {nameof(HttpsConnectionAdapterOptions.OnAuthenticate)} callback is not supported with HTTP/3.");
+            }
+
             // TODO Set other relevant values on options
             var sslServerAuthenticationOptions = new SslServerAuthenticationOptions
             {

src/Servers/Kestrel/test/InMemory.FunctionalTests/HttpsTests.cs

@@ -593,44 +593,44 @@ public async Task Http3_NoUseHttps_Throws()
         }
 
         [Fact]
-        public async Task Http3_UseHttp3Callback_NoSslServerOptions()
+        public void Http3_ServerOptionsSelectionCallback_Throws()
         {
             var serverOptions = CreateServerOptions();
             serverOptions.DefaultCertificate = _x509Certificate2;
 
-            IFeatureCollection bindFeatures = null;
-            var multiplexedConnectionListenerFactory = new MockMultiplexedConnectionListenerFactory();
-            multiplexedConnectionListenerFactory.OnBindAsync = (ep, features) =>
+            serverOptions.ListenLocalhost(5001, options =>
             {
-                bindFeatures = features;
-            };
-
-            var testContext = new TestServiceContext(LoggerFactory);
-            testContext.ServerOptions = serverOptions;
-            await using (var server = new TestServer(context => Task.CompletedTask,
-                testContext,
-                serverOptions =>
-                {
-                    serverOptions.ListenLocalhost(5001, listenOptions =>
+                options.Protocols = HttpProtocols.Http3;
+                var exception = Assert.Throws<NotSupportedException>(() =>
+                    options.UseHttps((SslStream stream, SslClientHelloInfo clientHelloInfo, object state, CancellationToken cancellationToken) =>
                     {
-                        listenOptions.Protocols = HttpProtocols.Http3;
-                        listenOptions.UseHttps((SslStream stream, SslClientHelloInfo clientHelloInfo, object state, CancellationToken cancellationToken) =>
-                        {
-                            return ValueTask.FromResult((new SslServerAuthenticationOptions()));
-                        }, state: null);
-                    });
-                },
-                services =>
-                {
-                    services.AddSingleton<IMultiplexedConnectionListenerFactory>(multiplexedConnectionListenerFactory);
-                }))
-            {
-            }
+                        return ValueTask.FromResult((new SslServerAuthenticationOptions()));
+                    }, state: null)
+                );
+                Assert.Equal("UseHttps with ServerOptionsSelectionCallback is not supported with HTTP/3.", exception.Message);
+            });
+        }
 
-            Assert.NotNull(bindFeatures);
+        [Fact]
+        public void Http3_TlsHandshakeCallbackOptions_Throws()
+        {
+            var serverOptions = CreateServerOptions();
+            serverOptions.DefaultCertificate = _x509Certificate2;
 
-            var sslOptions = bindFeatures.Get<SslServerAuthenticationOptions>();
-            Assert.Null(sslOptions);
+            serverOptions.ListenLocalhost(5001, options =>
+            {
+                options.Protocols = HttpProtocols.Http3;
+                var exception = Assert.Throws<NotSupportedException>(() =>
+                    options.UseHttps(new TlsHandshakeCallbackOptions()
+                    {
+                        OnConnection = context =>
+                        {
+                            return ValueTask.FromResult(new SslServerAuthenticationOptions());
+                        }
+                    })
+                );
+                Assert.Equal("UseHttps with TlsHandshakeCallbackOptions is not supported with HTTP/3.", exception.Message);
+            });
         }
 
         [Fact]

src/Servers/Kestrel/test/Interop.FunctionalTests/Http3/Http3TlsTests.cs

@@ -227,6 +227,33 @@ public async Task ClientCertificate_Allow_NotAvailable_Optional()
             await host.StopAsync().DefaultTimeout();
         }
 
+        [ConditionalFact]
+        [MsQuicSupported]
+        public async Task OnAuthentice_Available_Throws()
+        {
+            var builder = CreateHostBuilder(async context =>
+            {
+                await context.Response.WriteAsync("Hello World");
+            }, configureKestrel: kestrelOptions =>
+            {
+                kestrelOptions.ListenAnyIP(0, listenOptions =>
+                {
+                    listenOptions.Protocols = HttpProtocols.Http3;
+                    listenOptions.UseHttps(httpsOptions =>
+                    {
+                        httpsOptions.OnAuthenticate = (_, _) => { };
+                    });
+                });
+            });
+
+            using var host = builder.Build();
+            using var client = Http3Helpers.CreateClient();
+
+            var exception = await Assert.ThrowsAsync<NotSupportedException>(() =>
+                host.StartAsync().DefaultTimeout());
+            Assert.Equal("The OnAuthenticate callback is not supported with HTTP/3.", exception.Message);
+        }
+
         private IHostBuilder CreateHostBuilder(RequestDelegate requestDelegate, HttpProtocols? protocol = null, Action<KestrelServerOptions> configureKestrel = null)
         {
             return Http3Helpers.CreateHostBuilder(AddTestLogging, requestDelegate, protocol, configureKestrel);