Detect failed server connections in the bootstrap build

This change will cause our bootstrap build to fail if a threshold of server connection failures
occur.  The threshold is chosen a bit arbitrarily as valid failures can occur.  But 15 seems like
a number to warrant an investigation.  If this produces too many false positives it will be
adjusted appropriately.

src/Compilers/Core/MSBuildTask/AssemblyResolution.cs

@@ -59,9 +59,7 @@ private static bool TryRedirect(AssemblyName name)
 
             }
 
-#if DEBUG || BOOTSTRAP
-            ValidateBootstrap.AddFailedLoad(name);
-#endif
+            ValidateBootstrapUtil.AddFailedLoad(name);
 
             return false;
         }

src/Compilers/Core/MSBuildTask/ManagedCompiler.cs

@@ -540,6 +540,11 @@ private string LibDirectoryToUse()
         /// </summary>
         private int HandleResponse(BuildResponse response, string pathToTool, string responseFileCommands, string commandLineCommands)
         {
+            if (response.Type != BuildResponse.ResponseType.Completed)
+            {
+                ValidateBootstrapUtil.AddFailedServerConnection();
+            }
+
             switch (response.Type)
             {
                 case BuildResponse.ResponseType.Completed:

src/Compilers/Core/MSBuildTask/ValidateBootstrap.cs

@@ -6,18 +6,20 @@
 using System.Linq;
 using System.Reflection;
 using System;
+using System.Threading;
 
 namespace Microsoft.CodeAnalysis.BuildTasks
 {
 #if DEBUG || BOOTSTRAP
     /// <summary>
-    /// This task exists to help us validate our bootstrap building phase is executing correctly.  It 
-    /// is very easy for us, or MSBuild, to accidentally load DLLs from locations that we are 
-    /// not expecting.  This task takes steps to validate the bootstrap phase is executing as expected.
+    /// This task exists to help us validate our bootstrap building phase is executing correctly.  The bootstrap
+    /// phase of CI is the best way to validate the integration of our components is functioning correctly. Items
+    /// which are difficult to validate in a unit test scenario.
     /// </summary>
-    public sealed class ValidateBootstrap : Task
+    public sealed partial class ValidateBootstrap : Task
     {
         private static readonly ConcurrentDictionary<AssemblyName, byte> s_failedLoadSet = new ConcurrentDictionary<AssemblyName, byte>();
+        private static int s_failedServerConnectionCount = 0;
 
         private string _bootstrapPath;
 
@@ -69,6 +71,19 @@ public override bool Execute()
                 }
             }
 
+            // The number chosen is arbitrary here.  The goal of this check is to catch cases where a coding error has 
+            // broken our ability to use the compiler server in the bootstrap phase.
+            //
+            // It's possible on completely correct code for the server connection to fail.  There could be simply 
+            // named pipe errors, CPU load causing timeouts, etc ...  Hence flagging a single failure would produce
+            // a lot of false positives.  The current value was chosen as a reasonable number for warranting an 
+            // investigation.
+            if (s_failedServerConnectionCount > 20)
+            {
+                Log.LogError($"Too many compiler server connection failures detected: {s_failedServerConnectionCount}");
+                allGood = false;
+            }
+
             return allGood;
         }
 
@@ -104,6 +119,28 @@ internal static void AddFailedLoad(AssemblyName name)
                     break;
             }
         }
+
+        internal static void AddFailedServerConnection()
+        {
+            Interlocked.Increment(ref s_failedServerConnectionCount);
+        }
     }
 #endif
+
+    internal static class ValidateBootstrapUtil
+    {
+        internal static void AddFailedLoad(AssemblyName name)
+        {
+#if DEBUG || BOOTSTRAP
+            ValidateBootstrap.AddFailedLoad(name);
+#endif
+        }
+
+        internal static void AddFailedServerConnection()
+        {
+#if DEBUG || BOOTSTRAP
+            ValidateBootstrap.AddFailedServerConnection();
+#endif
+        }
+    }
 }