This is a partial fix for 1097123.

     The most immediate issue here is cancellation of the CancellationToken passed to AsyncQueue<T> will unconditionally call TaskCompletionSource.SetResult for the main Task value in the queue.  This cancellation can happen long after or in parallel with the completion of AsyncQueue<T>.  When this happens an exception is thrown.  This exception is causing the VBCSCompiler process to crash because it expects cancelling a token to be a safe operation.

     Longer term there are other issues in AsyncQueue<T> that need to be fixed (several race conditions).  Once this is checked in I willmove 1097123 to a Pri 1 and track the remaining fixes with that.   (changeset 1387407)

Src/Compilers/Core/CodeAnalysisTest/AsyncQueueTests.cs

+// Copyright (c) Microsoft Open Technologies, Inc.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using Microsoft.CodeAnalysis.Collections;
+using Microsoft.CodeAnalysis.Text;
+using Xunit;
+using Microsoft.CodeAnalysis.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Roslyn.Test.Utilities;
+
+namespace Microsoft.CodeAnalysis.UnitTests
+{
+    public class AsyncQueueTests
+    {
+        /// <summary>
+        /// Ensure that cancel after completion does not cause an exception to be thrown.
+        /// </summary>
+        [Fact]
+        [WorkItem(1097123, "DevDiv")]
+        public async Task CancelAfterCompleted()
+        {
+            var cts = new CancellationTokenSource();
+            var queue = new AsyncQueue<int>(cts.Token);
+            queue.Complete();
+            await queue.WhenCompletedAsync.ConfigureAwait(false);
+            Assert.Equal(TaskStatus.RanToCompletion, queue.WhenCompletedAsync.Status);
+            cts.Cancel();
+            Assert.Equal(TaskStatus.RanToCompletion, queue.WhenCompletedAsync.Status);
+        }
+    }
+}

Src/Compilers/Core/CodeAnalysisTest/CodeAnalysisTest.csproj

@@ -22,6 +22,7 @@
       <Link>FusionAssemblyIdentity.cs</Link>
     </Compile>
     <Compile Include="AnalyzerFileReferenceTests.cs" />
+    <Compile Include="AsyncQueueTests.cs" />
     <Compile Include="Diagnostics\DiagnosticLocalizationTests.cs" />
     <Compile Include="Emit\EmitOptionsTests.cs" />
     <Compile Include="Emit\CustomDebugInfoReaderTests.cs" />

Src/Compilers/Core/Portable/DiagnosticAnalyzer/AsyncQueue.TaskCompletionSourceWithCancellation.cs

@@ -12,7 +12,7 @@ public sealed partial class AsyncQueue<TElement>
         private sealed class TaskCompletionSourceWithCancellation<T> : TaskCompletionSource<T>
         {
             private readonly Action<object> OnCancelled = 
-                (tcs) => ((TaskCompletionSourceWithCancellation<T>)tcs).SetCanceled();
+                (tcs) => ((TaskCompletionSourceWithCancellation<T>)tcs).TrySetCanceled();
 
             private CancellationTokenRegistration cancellationTokenRegistration;