fixed issue where we crash due to our pending async work run after VS… (#34536)

* fixed issue where we crash due to our pending async work run after VS shutdown

this is another case where we have a pending async task that run after VS shutdown and it throws and our fail fast code catch that exception and crash VS.

general fix will be some thing like us making our fail fast code to aware shutdown situation and ignore any exception if we are in the shutdown situation.

but until we come up with proper design, this should handle one of high watson hits.

* PR feedbacks

src/VisualStudio/Core/Def/Implementation/Remote/ServiceHubRemoteHostClient.cs

@@ -7,6 +7,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.ErrorReporting;
 using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Internal.Log;
 using Microsoft.CodeAnalysis.Notification;
@@ -15,11 +16,10 @@
 using Microsoft.VisualStudio.Telemetry;
 using Roslyn.Utilities;
 using StreamJsonRpc;
+using Workspace = Microsoft.CodeAnalysis.Workspace;
 
 namespace Microsoft.VisualStudio.LanguageServices.Remote
 {
-    using Workspace = Microsoft.CodeAnalysis.Workspace;
-
     internal sealed partial class ServiceHubRemoteHostClient : RemoteHostClient
     {
         private enum GlobalNotificationState
@@ -31,6 +31,7 @@ private enum GlobalNotificationState
 
         private readonly JsonRpc _rpc;
         private readonly ConnectionManager _connectionManager;
+        private readonly CancellationTokenSource _shutdownCancellationTokenSource;
 
         /// <summary>
         /// Lock for the <see cref="_globalNotificationsTask"/> task chain.  Each time we hear 
@@ -128,6 +129,8 @@ public static async Task<ServiceHubRemoteHostClient> CreateWorkerAsync(Workspace
             Stream stream)
             : base(workspace)
         {
+            _shutdownCancellationTokenSource = new CancellationTokenSource();
+
             _connectionManager = connectionManager;
 
             _rpc = new JsonRpc(new JsonRpcMessageHandler(stream, stream), target: this);
@@ -154,6 +157,9 @@ protected override void OnStarted()
 
         protected override void OnStopped()
         {
+            // cancel all pending async work
+            _shutdownCancellationTokenSource.Cancel();
+
             // we are asked to stop. unsubscribe and dispose to disconnect.
             // there are 2 ways to get disconnected. one is Roslyn decided to disconnect with RemoteHost (ex, cancellation or recycle OOP) and
             // the other is external thing disconnecting remote host from us (ex, user killing OOP process).
@@ -161,8 +167,10 @@ protected override void OnStopped()
             // we don't need the event, otherwise, Disconnected event will be called twice.
             UnregisterGlobalOperationNotifications();
             UnregisterPersistentStorageLocationServiceChanges();
+
             _rpc.Disconnected -= OnRpcDisconnected;
             _rpc.Dispose();
+
             _connectionManager.Shutdown();
         }
 
@@ -210,12 +218,32 @@ private void UnregisterGlobalOperationNotifications()
             localTask.Wait();
         }
 
+        private async Task RpcInvokeAsync(string targetName, params object[] arguments)
+        {
+            // handle exception gracefully. don't crash VS due to this.
+            // especially on shutdown time. because of pending async BG work such as 
+            // OnGlobalOperationStarted and more, we can get into a situation where either
+            // we are in the middle of call when we are disconnected, or we runs
+            // after shutdown.
+            try
+            {
+                await _rpc.InvokeWithCancellationAsync(targetName, arguments, _shutdownCancellationTokenSource.Token).ConfigureAwait(false);
+            }
+            catch (Exception ex) when (ReportUnlessCanceled(ex))
+            {
+                if (!_shutdownCancellationTokenSource.IsCancellationRequested)
+                {
+                    RemoteHostCrashInfoBar.ShowInfoBar(Workspace);
+                }
+            }
+        }
+
         private void OnGlobalOperationStarted(object sender, EventArgs e)
         {
             lock (_globalNotificationsGate)
             {
-                _globalNotificationsTask = _globalNotificationsTask.ContinueWith(
-                    continuation, CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default).Unwrap();
+                _globalNotificationsTask = _globalNotificationsTask.SafeContinueWithFromAsync(
+                    continuation, _shutdownCancellationTokenSource.Token, TaskContinuationOptions.None, TaskScheduler.Default);
             }
 
             async Task<GlobalNotificationState> continuation(Task<GlobalNotificationState> previousTask)
@@ -227,8 +255,7 @@ async Task<GlobalNotificationState> continuation(Task<GlobalNotificationState> p
                     return previousTask.Result;
                 }
 
-                await _rpc.InvokeAsync(
-                    nameof(IRemoteHostService.OnGlobalOperationStarted), "").ConfigureAwait(false);
+                await RpcInvokeAsync(nameof(IRemoteHostService.OnGlobalOperationStarted), "").ConfigureAwait(false);
 
                 return GlobalNotificationState.Started;
             }
@@ -238,9 +265,8 @@ private void OnGlobalOperationStopped(object sender, GlobalOperationEventArgs e)
         {
             lock (_globalNotificationsGate)
             {
-                _globalNotificationsTask = _globalNotificationsTask.ContinueWith(
-                    continuation, CancellationToken.None,
-                    TaskContinuationOptions.None, TaskScheduler.Default).Unwrap();
+                _globalNotificationsTask = _globalNotificationsTask.SafeContinueWithFromAsync(
+                    continuation, _shutdownCancellationTokenSource.Token, TaskContinuationOptions.None, TaskScheduler.Default);
             }
 
             async Task<GlobalNotificationState> continuation(Task<GlobalNotificationState> previousTask)
@@ -252,9 +278,7 @@ async Task<GlobalNotificationState> continuation(Task<GlobalNotificationState> p
                     return previousTask.Result;
                 }
 
-                await _rpc.InvokeAsync(
-                    nameof(IRemoteHostService.OnGlobalOperationStopped),
-                    e.Operations, e.Cancelled).ConfigureAwait(false);
+                await RpcInvokeAsync(nameof(IRemoteHostService.OnGlobalOperationStopped), e.Operations, e.Cancelled).ConfigureAwait(false);
 
                 // Mark that we're stopped now.
                 return GlobalNotificationState.NotStarted;
@@ -288,10 +312,8 @@ private void EnqueueStorageLocationChange(SolutionId solutionId, string storageL
             {
                 _currentRemoteWorkspaceNotificationTask = _currentRemoteWorkspaceNotificationTask.SafeContinueWithFromAsync(_ =>
                 {
-                    return _rpc.InvokeAsync(
-                        nameof(IRemoteHostService.UpdateSolutionStorageLocation),
-                        new object[] { solutionId, storageLocation });
-                }, CancellationToken.None, TaskScheduler.Default);
+                    return RpcInvokeAsync(nameof(IRemoteHostService.UpdateSolutionStorageLocation), new object[] { solutionId, storageLocation });
+                }, _shutdownCancellationTokenSource.Token, TaskScheduler.Default);
             }
         }
 
@@ -311,5 +333,15 @@ private void OnRpcDisconnected(object sender, JsonRpcDisconnectedEventArgs e)
         {
             Stopped();
         }
+
+        private bool ReportUnlessCanceled(Exception ex)
+        {
+            if (_shutdownCancellationTokenSource.IsCancellationRequested)
+            {
+                return true;
+            }
+
+            return FatalError.ReportWithoutCrash(ex);
+        }
     }
 }