Account for Mutex ctor throwing

This changes the compiler server to be resilient in the face of the
Mutex constructor throwing an exception. It's documented as being able
to throw various exceptions but we never accounted for it. Had a real
world case pop up when using certain Docker containers on Linux.

In the case the Mutex ctor throws an exception just fall back to normal
csc / vbc execution. The build will complete just fine here, it just
won't have the performance boost of the compiler server.

closes #24124

src/Compilers/Shared/BuildServerConnection.cs

@@ -118,47 +118,62 @@ internal sealed class BuildServerConnection
             var clientDir = buildPaths.ClientDirectory;
             var timeoutNewProcess = timeoutOverride ?? TimeOutMsNewProcess;
             var timeoutExistingProcess = timeoutOverride ?? TimeOutMsExistingProcess;
-            var clientMutexName = GetClientMutexName(pipeName);
             Task<NamedPipeClientStream> pipeTask = null;
-            using (var clientMutex = new Mutex(initiallyOwned: true,
-                                               name: clientMutexName,
-                                               createdNew: out var holdsMutex))
+            Mutex clientMutex = null;
+            var holdsMutex = false;
+            try
             {
                 try
                 {
-                    if (!holdsMutex)
+                    var clientMutexName = GetClientMutexName(pipeName);
+                    clientMutex = new Mutex(initiallyOwned: true, name: clientMutexName, out holdsMutex);
+                }
+                catch
+                {
+                    // The Mutex constructor can throw in certain cases. One specific example is docker containers
+                    // where the /tmp directory is restricted. In those cases there is no reliable way to execute
+                    // the server and we need to fall back to the command line.
+                    //
+                    // Example: https://github.com/dotnet/roslyn/issues/24124
+                    return new RejectedBuildResponse();
+                }
+
+                if (!holdsMutex)
+                {
+                    try
                     {
-                        try
-                        {
-                            holdsMutex = clientMutex.WaitOne(timeoutNewProcess);
+                        holdsMutex = clientMutex.WaitOne(timeoutNewProcess);
 
-                            if (!holdsMutex)
-                            {
-                                return new RejectedBuildResponse();
-                            }
-                        }
-                        catch (AbandonedMutexException)
+                        if (!holdsMutex)
                         {
-                            holdsMutex = true;
+                            return new RejectedBuildResponse();
                         }
                     }
-
-                    // Check for an already running server
-                    var serverMutexName = GetServerMutexName(pipeName);
-                    bool wasServerRunning = WasServerMutexOpen(serverMutexName);
-                    var timeout = wasServerRunning ? timeoutExistingProcess : timeoutNewProcess;
-
-                    if (wasServerRunning || tryCreateServerFunc(clientDir, pipeName))
+                    catch (AbandonedMutexException)
                     {
-                        pipeTask = TryConnectToServerAsync(pipeName, timeout, cancellationToken);
+                        holdsMutex = true;
                     }
                 }
-                finally
+
+                // Check for an already running server
+                var serverMutexName = GetServerMutexName(pipeName);
+                bool wasServerRunning = WasServerMutexOpen(serverMutexName);
+                var timeout = wasServerRunning ? timeoutExistingProcess : timeoutNewProcess;
+
+                if (wasServerRunning || tryCreateServerFunc(clientDir, pipeName))
+                {
+                    pipeTask = TryConnectToServerAsync(pipeName, timeout, cancellationToken);
+                }
+            }
+            finally
+            {
+                if (clientMutex != null)
                 {
                     if (holdsMutex)
                     {
                         clientMutex.ReleaseMutex();
                     }
+                    clientMutex.Dispose();
                 }
             }
 
@@ -555,12 +570,21 @@ internal static string GetBasePipeName(string compilerExeDirectory)
 
         internal static bool WasServerMutexOpen(string mutexName)
         {
-            Mutex mutex;
-            var open = Mutex.TryOpenExisting(mutexName, out mutex);
-            if (open)
+            try
             {
-                mutex.Dispose();
-                return true;
+                Mutex mutex;
+                var open = Mutex.TryOpenExisting(mutexName, out mutex);
+                if (open)
+                {
+                    mutex.Dispose();
+                    return true;
+                }
+            }
+            catch
+            {
+                // In the case an exception occured trying to open the Mutex then 
+                // the assumption is that it's not open. 
+                return false;
             }
 
             return false;