Use FirstPipeInstance setting in named pipes transport

src/Servers/Kestrel/Transport.NamedPipes/src/Internal/NamedPipeConnectionListener.cs

@@ -27,25 +27,24 @@ internal sealed class NamedPipeConnectionListener : IConnectionListener
     private readonly MemoryPool<byte> _memoryPool;
     private readonly PipeOptions _inputOptions;
     private readonly PipeOptions _outputOptions;
-    private readonly Mutex _mutex;
+    private readonly NamedPipeServerStreamPoolPolicy _poolPolicy;
     private Task? _completeListeningTask;
     private int _disposed;
 
     public NamedPipeConnectionListener(
         NamedPipeEndPoint endpoint,
         NamedPipeTransportOptions options,
         ILoggerFactory loggerFactory,
-        ObjectPoolProvider objectPoolProvider,
-        Mutex mutex)
+        ObjectPoolProvider objectPoolProvider)
     {
         _log = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.NamedPipes");
         _endpoint = endpoint;
         _options = options;
-        _mutex = mutex;
         _memoryPool = options.MemoryPoolFactory();
         _listeningToken = _listeningTokenSource.Token;
         // Have to create the pool here (instead of DI) because the pool is specific to an endpoint.
-        _namedPipeServerStreamPool = objectPoolProvider.Create(new NamedPipeServerStreamPoolPolicy(endpoint, options));
+        _poolPolicy = new NamedPipeServerStreamPoolPolicy(endpoint, options);
+        _namedPipeServerStreamPool = objectPoolProvider.Create(_poolPolicy);
 
         // The OS maintains a backlog of clients that are waiting to connect, so the app queue only stores a single connection.
         // We want to have a queue plus a background task that populates the queue, rather than creating NamedPipeServerStream
@@ -77,6 +76,7 @@ public void Start()
         {
             // Start first stream inline to catch creation errors.
             var initialStream = _namedPipeServerStreamPool.Get();
+            _poolPolicy.SetFirstPipeStarted();
 
             listeningTasks[i] = Task.Run(() => StartAsync(initialStream));
         }
@@ -170,7 +170,6 @@ public async ValueTask DisposeAsync()
         }
 
         _listeningTokenSource.Dispose();
-        _mutex.Dispose();
         if (_completeListeningTask != null)
         {
             await _completeListeningTask;
@@ -185,6 +184,7 @@ private sealed class NamedPipeServerStreamPoolPolicy : IPooledObjectPolicy<Named
     {
         private readonly NamedPipeEndPoint _endpoint;
         private readonly NamedPipeTransportOptions _options;
+        private bool _hasFirstPipeStarted;
 
         public NamedPipeServerStreamPoolPolicy(NamedPipeEndPoint endpoint, NamedPipeTransportOptions options)
         {
@@ -196,6 +196,14 @@ public NamedPipeServerStream Create()
         {
             NamedPipeServerStream stream;
             var pipeOptions = NamedPipeOptions.Asynchronous | NamedPipeOptions.WriteThrough;
+            if (!_hasFirstPipeStarted)
+            {
+                // The first server stream created should validate that no one else is listening with a given name.
+                // Only the first server stream should make this test. The listener will almost always create multiple streams
+                // to listen on multiple threads and to handle parallel requests. The pool policy must be updated that the
+                // setting isn't needed after the first stream.
+                pipeOptions |= NamedPipeOptions.FirstPipeInstance;
+            }
             if (_options.CurrentUserOnly)
             {
                 pipeOptions |= NamedPipeOptions.CurrentUserOnly;
@@ -228,5 +236,10 @@ public NamedPipeServerStream Create()
         }
 
         public bool Return(NamedPipeServerStream obj) => !obj.IsConnected;
+
+        public void SetFirstPipeStarted()
+        {
+            _hasFirstPipeStarted = true;
+        }
     }
 }

src/Servers/Kestrel/Transport.NamedPipes/src/Internal/NamedPipeTransportFactory.cs

@@ -45,19 +45,22 @@ public ValueTask<IConnectionListener> BindAsync(EndPoint endpoint, CancellationT
             throw new NotSupportedException($@"Server name '{namedPipeEndPoint.ServerName}' is invalid. The server name must be ""{LocalComputerServerName}"".");
         }
 
-        // Creating a named pipe server with an name isn't exclusive. Create a mutex with the pipe name to prevent multiple endpoints
-        // accidently sharing the same pipe name. Will detect across Kestrel processes.
-        // Note that this doesn't prevent other applications from using the pipe name.
-        var mutexName = "Kestrel-NamedPipe-" + namedPipeEndPoint.PipeName;
-        var mutex = new Mutex(false, mutexName, out var createdNew);
-        if (!createdNew)
+        var listener = new NamedPipeConnectionListener(namedPipeEndPoint, _options, _loggerFactory, _objectPoolProvider);
+
+        // Start the listener and create NamedPipeServerStream instances immediately.
+        // The first server stream is created with the FirstPipeInstance flag. The FirstPipeInstance flag ensures
+        // that no other apps have a running pipe with the configured name. This check is important because:
+        // 1. Some settings, such as ACL, are chosen by the first pipe to start with a given name.
+        // 2. It's easy to run two Kestrel app instances. Want to avoid two apps listening on the same pipe and creating confusion.
+        //    The second launched app instance should immediately fail with an error message, just like port binding conflicts.
+        try
         {
-            mutex.Dispose();
-            throw new AddressInUseException($"Named pipe '{namedPipeEndPoint.PipeName}' is already in use by Kestrel.");
+            listener.Start();
+        }
+        catch (UnauthorizedAccessException ex)
+        {
+            throw new AddressInUseException($"Named pipe '{namedPipeEndPoint.PipeName}' is already in use.", ex);
         }
-
-        var listener = new NamedPipeConnectionListener(namedPipeEndPoint, _options, _loggerFactory, _objectPoolProvider, mutex);
-        listener.Start();
 
         return new ValueTask<IConnectionListener>(listener);
     }