diff --git a/src/Microsoft.AspNetCore.Hosting/Internal/WebHost.cs b/src/Microsoft.AspNetCore.Hosting/Internal/WebHost.cs
index 09f159a6..f03437ab 100644
--- a/src/Microsoft.AspNetCore.Hosting/Internal/WebHost.cs
+++ b/src/Microsoft.AspNetCore.Hosting/Internal/WebHost.cs
@@ -191,7 +191,7 @@ private RequestDelegate BuildApplication()
 
                 return builder.Build();
             }
-            catch (Exception ex) when (_options.CaptureStartupErrors)
+            catch (Exception ex)
             {
                 // EnsureApplicationServices may have failed due to a missing or throwing Startup class.
                 if (_applicationServices == null)
@@ -199,13 +199,18 @@ private RequestDelegate BuildApplication()
                     _applicationServices = _applicationServiceCollection.BuildServiceProvider();
                 }
 
-                EnsureServer();
-
                 // Write errors to standard out so they can be retrieved when not in development mode.
                 Console.Out.WriteLine("Application startup exception: " + ex.ToString());
                 var logger = _applicationServices.GetRequiredService<ILogger<WebHost>>();
                 logger.ApplicationError(ex);
 
+                if (!_options.CaptureStartupErrors)
+                {
+                    throw;
+                }
+
+                EnsureServer();
+
                 // Generate an HTML error page.
                 var hostingEnv = _applicationServices.GetRequiredService<IHostingEnvironment>();
                 var showDetailedErrors = hostingEnv.IsDevelopment() || _options.DetailedErrors;
diff --git a/test/Microsoft.AspNetCore.Hosting.Tests/WebHostBuilderTests.cs b/test/Microsoft.AspNetCore.Hosting.Tests/WebHostBuilderTests.cs
index 583a231b..6353fbda 100644
--- a/test/Microsoft.AspNetCore.Hosting.Tests/WebHostBuilderTests.cs
+++ b/test/Microsoft.AspNetCore.Hosting.Tests/WebHostBuilderTests.cs
@@ -851,6 +851,45 @@ public async Task Build_DoesNotThrowIfUnloadableAssemblyNameInHostingStartupAsse
             }
         }
 
+        [Fact]
+        public void StartupErrorsAreLoggedIfCaptureStartupErrorsIsTrue()
+        {
+            var builder = CreateWebHostBuilder()
+                .CaptureStartupErrors(true)
+                .Configure(app =>
+                {
+                    throw new InvalidOperationException("Startup exception");
+                })
+                .UseServer(new TestServer());
+
+            using (var host = (WebHost)builder.Build())
+            {
+                var sink = host.Services.GetRequiredService<ITestSink>();
+                Assert.True(sink.Writes.Any(w => w.Exception?.Message == "Startup exception"));
+            }
+        }
+
+        [Fact]
+        public void StartupErrorsAreLoggedIfCaptureStartupErrorsIsFalse()
+        {
+            ITestSink testSink = null;
+
+            var builder = CreateWebHostBuilder()
+                .CaptureStartupErrors(false)
+                .Configure(app =>
+                {
+                    testSink = app.ApplicationServices.GetRequiredService<ITestSink>();
+
+                    throw new InvalidOperationException("Startup exception");
+                })
+                .UseServer(new TestServer());
+
+            Assert.Throws<InvalidOperationException>(() => builder.Build());
+
+            Assert.NotNull(testSink);
+            Assert.True(testSink.Writes.Any(w => w.Exception?.Message == "Startup exception"));
+        }
+
         [Fact]
         public void HostingStartupTypeCtorThrowsIfNull()
         {