Merge pull request #21817 from heejaechang/moreDiagnosticMessage

added more diagnostic info on OOP code.

src/EditorFeatures/TestUtilities/Remote/InProcRemostHostClient.cs

@@ -27,7 +27,7 @@ public static async Task<RemoteHostClient> CreateAsync(Workspace workspace, bool
             var inprocServices = new InProcRemoteServices(runCacheCleanup);
 
             var remoteHostStream = await inprocServices.RequestServiceAsync(WellKnownRemoteHostServices.RemoteHostService, cancellationToken).ConfigureAwait(false);
-            var remotableDataRpc = new RemotableDataJsonRpc(workspace, await inprocServices.RequestServiceAsync(WellKnownServiceHubServices.SnapshotService, cancellationToken).ConfigureAwait(false));
+            var remotableDataRpc = new RemotableDataJsonRpc(workspace, inprocServices.Logger, await inprocServices.RequestServiceAsync(WellKnownServiceHubServices.SnapshotService, cancellationToken).ConfigureAwait(false));
 
             var instance = new InProcRemoteHostClient(workspace, inprocServices, new ReferenceCountedDisposable<RemotableDataJsonRpc>(remotableDataRpc), remoteHostStream);
 
@@ -80,7 +80,7 @@ public void RegisterService(string name, Func<Stream, IServiceProvider, ServiceH
             // this is what consumer actually use to communicate information
             var serviceStream = await _inprocServices.RequestServiceAsync(serviceName, cancellationToken).ConfigureAwait(false);
 
-            return new JsonRpcConnection(callbackTarget, serviceStream, _remotableDataRpc.TryAddReference());
+            return new JsonRpcConnection(_inprocServices.Logger, callbackTarget, serviceStream, _remotableDataRpc.TryAddReference());
         }
 
         protected override void OnStarted()
@@ -148,6 +148,7 @@ public InProcRemoteServices(bool runCacheCleanup)
             }
 
             public AssetStorage AssetStorage => _serviceProvider.AssetStorage;
+            public TraceSource Logger { get; } = new TraceSource("Default");
 
             public void RegisterService(string name, Func<Stream, IServiceProvider, ServiceHubServiceBase> serviceCreator)
             {

src/EditorFeatures/TestUtilities/Remote/WatsonReporter.cs

@@ -2,13 +2,18 @@
 
 using System;
 
-namespace Microsoft.VisualStudio.LanguageServices.Implementation
+namespace Microsoft.CodeAnalysis.ErrorReporting
 {
     /// <summary>
     /// Mock to make test project build
     /// </summary>
     internal class WatsonReporter
     {
+        public static void Report(string description, Exception exception)
+        {
+            // do nothing
+        }
+
         public static void Report(string description, Exception exception, Func<IFaultUtility, int> callback)
         {
             // do nothing

src/VisualStudio/Core/Def/Implementation/Log/VisualStudioErrorLogger.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Composition;
 using Microsoft.CodeAnalysis.ErrorLogger;
+using Microsoft.CodeAnalysis.ErrorReporting;
 using Microsoft.CodeAnalysis.Host.Mef;
 using Microsoft.VisualStudio.Shell;
 using static Microsoft.CodeAnalysis.RoslynAssemblyHelper;

src/VisualStudio/Core/Def/Implementation/TaskList/ProjectExternalErrorReporter.cs

@@ -7,6 +7,7 @@
 using System.Runtime.InteropServices;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.ErrorReporting;
 using Microsoft.VisualStudio.LanguageServices.Implementation.ProjectSystem;
 using Microsoft.VisualStudio.LanguageServices.Implementation.ProjectSystem.Extensions;
 using Microsoft.VisualStudio.LanguageServices.Implementation.Venus;

src/VisualStudio/Core/Def/Implementation/Watson/WatsonReporter.cs

@@ -5,9 +5,21 @@
 using Microsoft.VisualStudio.LanguageServices.Telemetry;
 using Microsoft.VisualStudio.Telemetry;
 
-namespace Microsoft.VisualStudio.LanguageServices.Implementation
+namespace Microsoft.CodeAnalysis.ErrorReporting
 {
-    internal class WatsonReporter
+    /// <summary>
+    /// Controls whether or not we actually report the failure.
+    /// There are situations where we know we're in a bad state and any further reports are unlikely to be
+    /// helpful, so we shouldn't send them.
+    /// </summary>
+    internal static class WatsonDisabled
+    {
+        // we have it this way to make debugging easier since VS debugger can't reach
+        // static type with same fully qualified name in multiple dlls.
+        public static bool s_reportWatson = true;
+    }
+
+    internal static class WatsonReporter
     {
         /// <summary>
         /// The default callback to pass to <see cref="TelemetrySessionExtensions.PostFault(TelemetrySession, string, string, Exception, Func{IFaultUtility, int})"/>.
@@ -15,13 +27,6 @@ internal class WatsonReporter
         /// </summary>
         private static Func<IFaultUtility, int> s_defaultCallback = _ => 0;
 
-        /// <summary>
-        /// Controls whether or not we actually report the failure.
-        /// There are situations where we know we're in a bad state and any further reports are unlikely to be
-        /// helpful, so we shouldn't send them.
-        /// </summary>
-        private static bool s_reportWatson = true;
-
         /// <summary>
         /// Report Non-Fatal Watson
         /// </summary>
@@ -51,7 +56,7 @@ public static void Report(string description, Exception exception)
         /// CAB.</param>
         public static void Report(string description, Exception exception, Func<IFaultUtility, int> callback)
         {
-            if (!s_reportWatson)
+            if (!WatsonDisabled.s_reportWatson)
             {
                 return;
             }
@@ -72,7 +77,7 @@ public static void Report(string description, Exception exception, Func<IFaultUt
                 // Once we've encountered one OOM we're likely to see more. There will probably be other
                 // failures as a direct result of the OOM, as well. These aren't helpful so we should just
                 // stop reporting failures.
-                s_reportWatson = false;
+                WatsonDisabled.s_reportWatson = false;
             }
         }
     }

src/VisualStudio/Core/Next/Remote/JsonRpcClient.cs

@@ -9,7 +9,6 @@
 using Microsoft.CodeAnalysis.ErrorReporting;
 using Microsoft.CodeAnalysis.Extensions;
 using Microsoft.CodeAnalysis.Remote;
-using Microsoft.VisualStudio.LanguageServices.Implementation;
 using Roslyn.Utilities;
 using StreamJsonRpc;
 
@@ -21,17 +20,21 @@ namespace Microsoft.VisualStudio.LanguageServices.Remote
     /// </summary>
     internal abstract class JsonRpcEx : IDisposable
     {
+        private readonly TraceSource _logger;
         private readonly JsonRpc _rpc;
 
         private JsonRpcDisconnectedEventArgs _debuggingLastDisconnectReason;
         private string _debuggingLastDisconnectCallstack;
 
-        public JsonRpcEx(Stream stream, object callbackTarget, bool useThisAsCallback)
+        public JsonRpcEx(TraceSource logger, Stream stream, object callbackTarget, bool useThisAsCallback)
         {
+            Contract.Requires(logger != null);
             Contract.Requires(stream != null);
 
             var target = useThisAsCallback ? this : callbackTarget;
 
+            _logger = logger;
+
             _rpc = new JsonRpc(new JsonRpcMessageHandler(stream, stream), target);
             _rpc.JsonSerializer.Converters.Add(AggregateJsonConverter.Instance);
 
@@ -40,6 +43,11 @@ public JsonRpcEx(Stream stream, object callbackTarget, bool useThisAsCallback)
 
         protected abstract void Dispose(bool disposing);
 
+        protected virtual void Disconnected(JsonRpcDisconnectedEventArgs e)
+        {
+            // do nothing
+        }
+
         public async Task InvokeAsync(string targetName, IReadOnlyList<object> arguments, CancellationToken cancellationToken)
         {
             cancellationToken.ThrowIfCancellationRequested();
@@ -57,6 +65,8 @@ public async Task InvokeAsync(string targetName, IReadOnlyList<object> arguments
                 // the exception
                 cancellationToken.ThrowIfCancellationRequested();
 
+                LogError($"exception: {ex.ToString()}");
+
                 // this is to make us not crash. we should remove this once we figure out
                 // what is causing this
                 ThrowOwnCancellationToken();
@@ -79,6 +89,8 @@ public async Task<T> InvokeAsync<T>(string targetName, IReadOnlyList<object> arg
                 // the exception
                 cancellationToken.ThrowIfCancellationRequested();
 
+                LogError($"exception: {ex.ToString()}");
+
                 // this is to make us not crash. we should remove this once we figure out
                 // what is causing this
                 ThrowOwnCancellationToken();
@@ -86,16 +98,55 @@ public async Task<T> InvokeAsync<T>(string targetName, IReadOnlyList<object> arg
             }
         }
 
-        public Task InvokeAsync(
+        public async Task InvokeAsync(
             string targetName, IReadOnlyList<object> arguments, Func<Stream, CancellationToken, Task> funcWithDirectStreamAsync, CancellationToken cancellationToken)
         {
-            return Extensions.InvokeAsync(_rpc, targetName, arguments, funcWithDirectStreamAsync, cancellationToken);
+            cancellationToken.ThrowIfCancellationRequested();
+
+            try
+            {
+                await Extensions.InvokeAsync(_rpc, targetName, arguments, funcWithDirectStreamAsync, cancellationToken).ConfigureAwait(false);
+            }
+            catch (Exception ex) // no when since Extensions.InvokeAsync already recorded it
+            {
+                // any exception can be thrown from StreamJsonRpc if JsonRpc is disposed in the middle of read/write.
+                // until we move to newly added cancellation support in JsonRpc, we will catch exception and translate to
+                // cancellation exception here. if any exception is thrown unrelated to cancellation, then we will rethrow
+                // the exception
+                cancellationToken.ThrowIfCancellationRequested();
+
+                LogError($"exception: {ex.ToString()}");
+
+                // this is to make us not crash. we should remove this once we figure out
+                // what is causing this
+                ThrowOwnCancellationToken();
+            }
         }
 
-        public Task<T> InvokeAsync<T>(
+        public async Task<T> InvokeAsync<T>(
             string targetName, IReadOnlyList<object> arguments, Func<Stream, CancellationToken, Task<T>> funcWithDirectStreamAsync, CancellationToken cancellationToken)
         {
-            return Extensions.InvokeAsync(_rpc, targetName, arguments, funcWithDirectStreamAsync, cancellationToken);
+            cancellationToken.ThrowIfCancellationRequested();
+
+            try
+            {
+                return await Extensions.InvokeAsync(_rpc, targetName, arguments, funcWithDirectStreamAsync, cancellationToken).ConfigureAwait(false);
+            }
+            catch (Exception ex) // no when since Extensions.InvokeAsync already recorded it
+            {
+                // any exception can be thrown from StreamJsonRpc if JsonRpc is disposed in the middle of read/write.
+                // until we move to newly added cancellation support in JsonRpc, we will catch exception and translate to
+                // cancellation exception here. if any exception is thrown unrelated to cancellation, then we will rethrow
+                // the exception
+                cancellationToken.ThrowIfCancellationRequested();
+
+                LogError($"exception: {ex.ToString()}");
+
+                // this is to make us not crash. we should remove this once we figure out
+                // what is causing this
+                ThrowOwnCancellationToken();
+                return Contract.FailWithReturn<T>("can't reach here");
+            }
         }
 
         // these are for debugging purpose. once we find out root cause of the issue
@@ -114,7 +165,7 @@ private bool ReportUnlessCanceled(Exception ex, CancellationToken cancellationTo
             s_debuggingLastDisconnectCallstack = _debuggingLastDisconnectCallstack;
 
             // send NFW to figure out why this is happening
-            ReportExtraInfoAsNFW(ex);
+            ex.ReportServiceHubNFW("RemoteHost Failed");
 
             GC.KeepAlive(_debuggingLastDisconnectReason);
             GC.KeepAlive(_debuggingLastDisconnectCallstack);
@@ -144,6 +195,9 @@ private void ThrowOwnCancellationToken()
                     ServicesVSResources.Unfortunately_a_process_used_by_Visual_Studio_has_encountered_an_unrecoverable_error_We_recommend_saving_your_work_and_then_closing_and_restarting_Visual_Studio);
             }
 
+            // log disconnect information before throw
+            LogDisconnectInfo(_debuggingLastDisconnectReason, _debuggingLastDisconnectCallstack);
+
             // create its own cancellation token and throw it
             using (var ownCancellationSource = new CancellationTokenSource())
             {
@@ -152,49 +206,6 @@ private void ThrowOwnCancellationToken()
             }
         }
 
-        private void ReportExtraInfoAsNFW(Exception ex)
-        {
-            WatsonReporter.Report("RemoteHost Failed", ex, u =>
-            {
-                try
-                {
-                    // we will record dumps for all service hub processes
-                    foreach (var p in Process.GetProcessesByName("ServiceHub.RoslynCodeAnalysisService32"))
-                    {
-                        // include all remote host processes
-                        u.AddProcessDump(p.Id);
-                    }
-
-                    // include all service hub logs as well
-                    var logPath = Path.Combine(Path.GetTempPath(), "servicehub", "logs");
-                    if (Directory.Exists(logPath))
-                    {
-                        // attach all log files that are modified less than 1 day before.
-                        var now = DateTime.UtcNow;
-                        var oneDay = TimeSpan.FromDays(1);
-
-                        foreach (var file in Directory.EnumerateFiles(logPath, "*.log"))
-                        {
-                            var lastWrite = File.GetLastWriteTimeUtc(file);
-                            if (now - lastWrite > oneDay)
-                            {
-                                continue;
-                            }
-
-                            u.AddFile(file);
-                        }
-                    }
-                }
-                catch
-                {
-                    // ignore issue
-                }
-
-                // 0 means send watson
-                return 0;
-            });
-        }
-
         protected void Disconnect()
         {
             _rpc.Dispose();
@@ -207,11 +218,31 @@ protected void StartListening()
             _rpc.StartListening();
         }
 
-        protected virtual void OnDisconnected(object sender, JsonRpcDisconnectedEventArgs e)
+        protected void LogError(string message)
+        {
+            _logger.TraceEvent(TraceEventType.Error, 1, message);
+        }
+
+        protected void LogDisconnectInfo(JsonRpcDisconnectedEventArgs e, string callstack)
+        {
+            if (e != null)
+            {
+                LogError($"disconnect exception: {e.Description}, {e.Reason}, {e.LastMessage}, {e.Exception?.ToString()}");
+            }
+
+            if (callstack != null)
+            {
+                LogError($"disconnect callstack: {callstack}");
+            }
+        }
+
+        private void OnDisconnected(object sender, JsonRpcDisconnectedEventArgs e)
         {
-            // do nothing
             _debuggingLastDisconnectReason = e;
             _debuggingLastDisconnectCallstack = new StackTrace().ToString();
+
+            // tell we got disconnected
+            Disconnected(e);
         }
 
         public void Dispose()

src/VisualStudio/Core/Next/Remote/JsonRpcConnection.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
@@ -20,13 +21,14 @@ internal class JsonRpcConnection : RemoteHostClient.Connection
         private readonly ReferenceCountedDisposable<RemotableDataJsonRpc> _remoteDataRpc;
 
         public JsonRpcConnection(
+            TraceSource logger,
             object callbackTarget,
             Stream serviceStream,
             ReferenceCountedDisposable<RemotableDataJsonRpc> dataRpc)
         {
             Contract.ThrowIfNull(dataRpc);
 
-            _serviceRpc = new ServiceJsonRpcEx(serviceStream, callbackTarget);
+            _serviceRpc = new ServiceJsonRpcEx(logger, serviceStream, callbackTarget);
             _remoteDataRpc = dataRpc;
         }
 
@@ -73,8 +75,8 @@ private sealed class ServiceJsonRpcEx : JsonRpcEx
         {
             private readonly object _callbackTarget;
 
-            public ServiceJsonRpcEx(Stream stream, object callbackTarget)
-                : base(stream, callbackTarget, useThisAsCallback: false)
+            public ServiceJsonRpcEx(TraceSource logger, Stream stream, object callbackTarget)
+                : base(logger, stream, callbackTarget, useThisAsCallback: false)
             {
                 // this one doesn't need cancellation token since it has nothing to cancel
                 _callbackTarget = callbackTarget;

src/VisualStudio/Core/Next/Remote/RemotableDataJsonRpcEx.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Execution;
+using Microsoft.CodeAnalysis.Internal.Log;
 using Microsoft.CodeAnalysis.Remote;
 using Roslyn.Utilities;
 using StreamJsonRpc;
-using Microsoft.CodeAnalysis.Internal.Log;
 
 namespace Microsoft.VisualStudio.LanguageServices.Remote
 {
@@ -27,8 +28,8 @@ internal sealed class RemotableDataJsonRpc : JsonRpcEx
         private readonly IRemotableDataService _remotableDataService;
         private readonly CancellationTokenSource _shutdownCancellationSource;
 
-        public RemotableDataJsonRpc(Microsoft.CodeAnalysis.Workspace workspace, Stream stream)
-            : base(stream, callbackTarget: null, useThisAsCallback: true)
+        public RemotableDataJsonRpc(Microsoft.CodeAnalysis.Workspace workspace, TraceSource logger, Stream stream)
+            : base(logger, stream, callbackTarget: null, useThisAsCallback: true)
         {
             _remotableDataService = workspace.Services.GetService<IRemotableDataService>();
 
@@ -58,15 +59,34 @@ public async Task RequestAssetAsync(int scopeId, Checksum[] checksums, string st
                     await stream.FlushAsync(source.Token).ConfigureAwait(false);
                 }
             }
-            catch (IOException)
+            catch (Exception ex) when (ReportUnlessCanceled(ex, cancellationToken))
             {
-                // direct stream can throw if cancellation happens since direct stream still uses
-                // disconnection for cancellation
+                // only expected exception will be catched. otherwise, NFW and let it propagate
+                Debug.Assert(cancellationToken.IsCancellationRequested || ex is IOException);
             }
-            catch (OperationCanceledException)
+        }
+
+        private bool ReportUnlessCanceled(Exception ex, CancellationToken cancellationToken)
+        {
+            if (cancellationToken.IsCancellationRequested)
             {
-                // this can happen if connection got shutdown
+                // any exception can happen if things are cancelled.
+                return true;
             }
+
+            if (ex is IOException)
+            {
+                // direct connection can be disconnected before cancellation token from remote host have
+                // passed to us
+                return true;
+            }
+
+            // log the exception
+            LogError("unexpected exception from RequestAsset: " + ex.ToString());
+
+            // report NFW
+            ex.ReportServiceHubNFW("RequestAssetFailed");
+            return false;
         }
 
         private async Task WriteAssetAsync(ObjectWriter writer, int scopeId, Checksum[] checksums, CancellationToken cancellationToken)
@@ -124,8 +144,14 @@ protected override void Dispose(bool disposing)
             Disconnect();
         }
 
-        protected override void OnDisconnected(object sender, JsonRpcDisconnectedEventArgs e)
+        protected override void Disconnected(JsonRpcDisconnectedEventArgs e)
         {
+            if (e.Reason != DisconnectedReason.Disposed)
+            {
+                // log when this happens
+                LogDisconnectInfo(e, new StackTrace().ToString());
+            }
+
             _shutdownCancellationSource.Cancel();
         }
     }

src/VisualStudio/Core/Next/Remote/ServiceHubRemoteHostClient.cs

@@ -11,7 +11,6 @@
 using Microsoft.CodeAnalysis.Notification;
 using Microsoft.CodeAnalysis.Remote;
 using Microsoft.ServiceHub.Client;
-using Microsoft.VisualStudio.LanguageServices.Implementation;
 using Microsoft.VisualStudio.LanguageServices.Implementation.ProjectSystem;
 using Microsoft.VisualStudio.Telemetry;
 using Roslyn.Utilities;
@@ -64,7 +63,7 @@ private enum GlobalNotificationState
                 var timeout = TimeSpan.FromMilliseconds(workspace.Options.GetOption(RemoteHostOptions.RequestServiceTimeoutInMS));
                 var remoteHostStream = await RequestServiceAsync(primary, WellKnownRemoteHostServices.RemoteHostService, hostGroup, timeout, cancellationToken).ConfigureAwait(false);
 
-                var remotableDataRpc = new RemotableDataJsonRpc(workspace, await RequestServiceAsync(primary, WellKnownServiceHubServices.SnapshotService, hostGroup, timeout, cancellationToken).ConfigureAwait(false));
+                var remotableDataRpc = new RemotableDataJsonRpc(workspace, primary.Logger, await RequestServiceAsync(primary, WellKnownServiceHubServices.SnapshotService, hostGroup, timeout, cancellationToken).ConfigureAwait(false));
                 var instance = new ServiceHubRemoteHostClient(workspace, primary, hostGroup, new ReferenceCountedDisposable<RemotableDataJsonRpc>(remotableDataRpc), remoteHostStream);
 
                 // make sure connection is done right
@@ -117,7 +116,7 @@ internal static async Task RegisterWorkspaceHostAsync(Workspace workspace, Remot
             HubClient hubClient,
             HostGroup hostGroup,
             ReferenceCountedDisposable<RemotableDataJsonRpc> remotableDataRpc,
-            Stream stream) 
+            Stream stream)
             : base(workspace)
         {
             Contract.ThrowIfNull(remotableDataRpc);
@@ -152,7 +151,7 @@ public override async Task<Connection> TryCreateConnectionAsync(string serviceNa
             // this is what consumer actually use to communicate information
             var serviceStream = await RequestServiceAsync(_hubClient, serviceName, _hostGroup, _timeout, cancellationToken).ConfigureAwait(false);
 
-            return new JsonRpcConnection(callbackTarget, serviceStream, dataRpc);
+            return new JsonRpcConnection(_hubClient.Logger, callbackTarget, serviceStream, dataRpc);
         }
 
         protected override void OnStarted()
@@ -293,7 +292,7 @@ private void OnRpcDisconnected(object sender, JsonRpcDisconnectedEventArgs e)
                     {
                         // RequestServiceAsync should never fail unless service itself is actually broken.
                         // So far, we catched multiple issues from this NFW. so we will keep this NFW.
-                        WatsonReporter.Report("RequestServiceAsync Failed", ex, ReportDetailInfo);
+                        ex.ReportServiceHubNFW("RequestServiceAsync Failed");
 
                         lastException = ex;
                     }
@@ -346,53 +345,5 @@ private static async Task<Stream> RequestServiceAsync(HubClient client, ServiceD
             // request service to HubClient timed out, more than we are willing to wait
             throw new TimeoutException("RequestServiceAsync timed out");
         }
-
-        private static int ReportDetailInfo(IFaultUtility faultUtility)
-        {
-            // 0 means send watson, otherwise, cancel watson
-            // we always send watson since dump itself can have valuable data
-            var exitCode = 0;
-
-            try
-            {
-                var logPath = Path.Combine(Path.GetTempPath(), "servicehub", "logs");
-                if (!Directory.Exists(logPath))
-                {
-                    return exitCode;
-                }
-
-                // attach all log files that are modified less than 1 day before.
-                var now = DateTime.UtcNow;
-                var oneDay = TimeSpan.FromDays(1);
-
-                foreach (var file in Directory.EnumerateFiles(logPath, "*.log"))
-                {
-                    var lastWrite = File.GetLastWriteTimeUtc(file);
-                    if (now - lastWrite > oneDay)
-                    {
-                        continue;
-                    }
-
-                    faultUtility.AddFile(file);
-                }
-            }
-            catch (Exception ex) when (ReportNonIOException(ex))
-            {
-            }
-
-            return exitCode;
-        }
-
-        private static bool ReportNonIOException(Exception ex)
-        {
-            // IOException is expected. log other exceptions
-            if (!(ex is IOException))
-            {
-                WatsonReporter.Report(ex);
-            }
-
-            // catch all exception. not worth crashing VS.
-            return true;
-        }
     }
 }

src/VisualStudio/InteractiveServices/Interactive/VsInteractiveWindowPackage.cs

@@ -15,6 +15,7 @@
 using Microsoft.VisualStudio.InteractiveWindow;
 using System.Reflection;
 using Microsoft.VisualStudio.InteractiveWindow.Shell;
+using Microsoft.CodeAnalysis.ErrorReporting;
 
 namespace Microsoft.VisualStudio.LanguageServices.Interactive
 {

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_AddImport.cs

@@ -18,22 +18,25 @@ internal partial class CodeAnalysisService : IRemoteAddImportFeatureService
             DocumentId documentId, TextSpan span, string diagnosticId, bool placeSystemNamespaceFirst,
             bool searchReferenceAssemblies, IList<PackageSource> packageSources, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return await RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
-                var document = solution.GetDocument(documentId);
+                using (UserOperationBooster.Boost())
+                {
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                    var document = solution.GetDocument(documentId);
 
-                var service = document.GetLanguageService<IAddImportFeatureService>();
+                    var service = document.GetLanguageService<IAddImportFeatureService>();
 
-                var symbolSearchService = new SymbolSearchService(this);
+                    var symbolSearchService = new SymbolSearchService(this);
 
-                var result = await service.GetFixesAsync(
-                    document, span, diagnosticId, placeSystemNamespaceFirst,
-                    symbolSearchService, searchReferenceAssemblies,
-                    packageSources.ToImmutableArray(), cancellationToken).ConfigureAwait(false);
+                    var result = await service.GetFixesAsync(
+                        document, span, diagnosticId, placeSystemNamespaceFirst,
+                        symbolSearchService, searchReferenceAssemblies,
+                        packageSources.ToImmutableArray(), cancellationToken).ConfigureAwait(false);
 
-                return result;
-            }
+                    return result;
+                }
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         /// <summary>

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_CodeLens.cs

@@ -13,9 +13,9 @@ namespace Microsoft.CodeAnalysis.Remote
 {
     internal partial class CodeAnalysisService
     {
-        public async Task<ReferenceCount> GetReferenceCountAsync(DocumentId documentId, TextSpan textSpan, int maxResultCount, CancellationToken cancellationToken)
+        public Task<ReferenceCount> GetReferenceCountAsync(DocumentId documentId, TextSpan textSpan, int maxResultCount, CancellationToken cancellationToken)
         {
-            try
+            return RunServiceAsync(async () =>
             {
                 using (Internal.Log.Logger.LogBlock(FunctionId.CodeAnalysisService_GetReferenceCountAsync, documentId.ProjectId.DebugName, cancellationToken))
                 {
@@ -25,25 +25,12 @@ public async Task<ReferenceCount> GetReferenceCountAsync(DocumentId documentId,
                     return await CodeLensReferencesServiceFactory.Instance.GetReferenceCountAsync(solution, documentId,
                         syntaxNode, maxResultCount, cancellationToken).ConfigureAwait(false);
                 }
-            }
-            catch (IOException)
-            {
-                // stream to send over result has closed before we
-                // had chance to check cancellation
-            }
-            catch (OperationCanceledException)
-            {
-                // rpc connection has closed.
-                // this can happen if client side cancelled the
-                // operation
-            }
-
-            return null;
+            }, cancellationToken);
         }
 
-        public async Task<IEnumerable<ReferenceLocationDescriptor>> FindReferenceLocationsAsync(DocumentId documentId, TextSpan textSpan, CancellationToken cancellationToken)
+        public Task<IEnumerable<ReferenceLocationDescriptor>> FindReferenceLocationsAsync(DocumentId documentId, TextSpan textSpan, CancellationToken cancellationToken)
         {
-            try
+            return RunServiceAsync(async () =>
             {
                 using (Internal.Log.Logger.LogBlock(FunctionId.CodeAnalysisService_FindReferenceLocationsAsync, documentId.ProjectId.DebugName, cancellationToken))
                 {
@@ -53,25 +40,12 @@ public async Task<IEnumerable<ReferenceLocationDescriptor>> FindReferenceLocatio
                     return await CodeLensReferencesServiceFactory.Instance.FindReferenceLocationsAsync(solution, documentId,
                         syntaxNode, cancellationToken).ConfigureAwait(false);
                 }
-            }
-            catch (IOException)
-            {
-                // stream to send over result has closed before we
-                // had chance to check cancellation
-            }
-            catch (OperationCanceledException)
-            {
-                // rpc connection has closed.
-                // this can happen if client side cancelled the
-                // operation
-            }
-
-            return null;
+            }, cancellationToken);
         }
 
-        public async Task<IEnumerable<ReferenceMethodDescriptor>> FindReferenceMethodsAsync(DocumentId documentId, TextSpan textSpan, CancellationToken cancellationToken)
+        public Task<IEnumerable<ReferenceMethodDescriptor>> FindReferenceMethodsAsync(DocumentId documentId, TextSpan textSpan, CancellationToken cancellationToken)
         {
-            try
+            return RunServiceAsync(async () =>
             {
                 using (Internal.Log.Logger.LogBlock(FunctionId.CodeAnalysisService_FindReferenceMethodsAsync, documentId.ProjectId.DebugName, cancellationToken))
                 {
@@ -81,25 +55,12 @@ public async Task<IEnumerable<ReferenceMethodDescriptor>> FindReferenceMethodsAs
                     return await CodeLensReferencesServiceFactory.Instance.FindReferenceMethodsAsync(solution, documentId,
                         syntaxNode, cancellationToken).ConfigureAwait(false);
                 }
-            }
-            catch (IOException)
-            {
-                // stream to send over result has closed before we
-                // had chance to check cancellation
-            }
-            catch (OperationCanceledException)
-            {
-                // rpc connection has closed.
-                // this can happen if client side cancelled the
-                // operation
-            }
-
-            return null;
+            }, cancellationToken);
         }
 
-        public async Task<string> GetFullyQualifiedName(DocumentId documentId, TextSpan textSpan, CancellationToken cancellationToken)
+        public Task<string> GetFullyQualifiedName(DocumentId documentId, TextSpan textSpan, CancellationToken cancellationToken)
         {
-            try
+            return RunServiceAsync(async () =>
             {
                 using (Internal.Log.Logger.LogBlock(FunctionId.CodeAnalysisService_GetFullyQualifiedName, documentId.ProjectId.DebugName, cancellationToken))
                 {
@@ -109,20 +70,7 @@ public async Task<string> GetFullyQualifiedName(DocumentId documentId, TextSpan
                     return await CodeLensReferencesServiceFactory.Instance.GetFullyQualifiedName(solution, documentId,
                         syntaxNode, cancellationToken).ConfigureAwait(false);
                 }
-            }
-            catch (IOException)
-            {
-                // stream to send over result has closed before we
-                // had chance to check cancellation
-            }
-            catch (OperationCanceledException)
-            {
-                // rpc connection has closed.
-                // this can happen if client side cancelled the
-                // operation
-            }
-
-            return null;
+            }, cancellationToken);
         }
     }
 }

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_DesignerAttributes.cs

@@ -19,23 +19,26 @@ internal partial class CodeAnalysisService : IRemoteDesignerAttributeService
         /// 
         /// This will be called by ServiceHub/JsonRpc framework
         /// </summary>
-        public async Task<IList<DesignerAttributeDocumentData>> ScanDesignerAttributesAsync(ProjectId projectId, CancellationToken cancellationToken)
+        public Task<IList<DesignerAttributeDocumentData>> ScanDesignerAttributesAsync(ProjectId projectId, CancellationToken cancellationToken)
         {
-            using (RoslynLogger.LogBlock(FunctionId.CodeAnalysisService_GetDesignerAttributesAsync, projectId.DebugName, cancellationToken))
+            return RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
-                var project = solution.GetProject(projectId);
+                using (RoslynLogger.LogBlock(FunctionId.CodeAnalysisService_GetDesignerAttributesAsync, projectId.DebugName, cancellationToken))
+                {
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                    var project = solution.GetProject(projectId);
 
-                var data = await AbstractDesignerAttributeService.TryAnalyzeProjectInCurrentProcessAsync(
-                    project, cancellationToken).ConfigureAwait(false);
+                    var data = await AbstractDesignerAttributeService.TryAnalyzeProjectInCurrentProcessAsync(
+                        project, cancellationToken).ConfigureAwait(false);
 
-                if (data.Count == 0)
-                {
-                    return SpecializedCollections.EmptyList<DesignerAttributeDocumentData>();
-                }
+                    if (data.Count == 0)
+                    {
+                        return SpecializedCollections.EmptyList<DesignerAttributeDocumentData>();
+                    }
 
-                return data.Values.ToList();
-            }
+                    return data.Values.ToList();
+                }
+            }, cancellationToken);
         }
     }
 }

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_Diagnostics.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
+using System.Collections.Concurrent;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
@@ -21,32 +22,35 @@ internal partial class CodeAnalysisService
         /// 
         /// This will be called by ServiceHub/JsonRpc framework
         /// </summary>
-        public async Task CalculateDiagnosticsAsync(DiagnosticArguments arguments, string streamName, CancellationToken cancellationToken)
+        public Task CalculateDiagnosticsAsync(DiagnosticArguments arguments, string streamName, CancellationToken cancellationToken)
         {
-            // if this analysis is explicitly asked by user, boost priority of this request
-            using (RoslynLogger.LogBlock(FunctionId.CodeAnalysisService_CalculateDiagnosticsAsync, arguments.ProjectId.DebugName, cancellationToken))
-            using (arguments.ForcedAnalysis ? UserOperationBooster.Boost() : default)
+            return RunServiceAsync(async () =>
             {
-                try
+                // if this analysis is explicitly asked by user, boost priority of this request
+                using (RoslynLogger.LogBlock(FunctionId.CodeAnalysisService_CalculateDiagnosticsAsync, arguments.ProjectId.DebugName, cancellationToken))
+                using (arguments.ForcedAnalysis ? UserOperationBooster.Boost() : default)
                 {
-                    // entry point for diagnostic service
-                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                    try
+                    {
+                        // entry point for diagnostic service
+                        var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
 
-                    var optionSet = await RoslynServices.AssetService.GetAssetAsync<OptionSet>(arguments.OptionSetChecksum, cancellationToken).ConfigureAwait(false);
-                    var projectId = arguments.ProjectId;
-                    var analyzers = RoslynServices.AssetService.GetGlobalAssetsOfType<AnalyzerReference>(cancellationToken);
+                        var optionSet = await RoslynServices.AssetService.GetAssetAsync<OptionSet>(arguments.OptionSetChecksum, cancellationToken).ConfigureAwait(false);
+                        var projectId = arguments.ProjectId;
+                        var analyzers = RoslynServices.AssetService.GetGlobalAssetsOfType<AnalyzerReference>(cancellationToken);
 
-                    var result = await (new DiagnosticComputer(solution.GetProject(projectId))).GetDiagnosticsAsync(
-                        analyzers, optionSet, arguments.AnalyzerIds, arguments.ReportSuppressedDiagnostics, arguments.LogAnalyzerExecutionTime, cancellationToken).ConfigureAwait(false);
+                        var result = await (new DiagnosticComputer(solution.GetProject(projectId))).GetDiagnosticsAsync(
+                            analyzers, optionSet, arguments.AnalyzerIds, arguments.ReportSuppressedDiagnostics, arguments.LogAnalyzerExecutionTime, cancellationToken).ConfigureAwait(false);
 
-                    await SerializeDiagnosticResultAsync(streamName, result, cancellationToken).ConfigureAwait(false);
+                        await SerializeDiagnosticResultAsync(streamName, result, cancellationToken).ConfigureAwait(false);
+                    }
+                    catch (IOException)
+                    {
+                        // direct stream to send over result has closed before we
+                        // had chance to check cancellation
+                    }
                 }
-                catch (IOException)
-                {
-                    // direct stream to send over result has closed before we
-                    // had chance to check cancellation
-                }
-            }
+            }, cancellationToken);
         }
 
         private async Task SerializeDiagnosticResultAsync(string streamName, DiagnosticAnalysisResultMap<string, DiagnosticAnalysisResultBuilder> result, CancellationToken cancellationToken)

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_DocumentHighlights.cs

@@ -17,20 +17,23 @@ internal partial class CodeAnalysisService : IRemoteDocumentHighlights
         public async Task<IList<SerializableDocumentHighlights>> GetDocumentHighlightsAsync(
             DocumentId documentId, int position, DocumentId[] documentIdsToSearch, CancellationToken cancellationToken)
         {
-            // NOTE: In projection scenarios, we might get a set of documents to search
-            // that are not all the same language and might not exist in the OOP process
-            // (like the JS parts of a .cshtml file). Filter them out here.  This will
-            // need to be revisited if we someday support FAR between these languages.
-            var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
-            var document = solution.GetDocument(documentId);
-            var documentsToSearch = ImmutableHashSet.CreateRange(
-                documentIdsToSearch.Select(solution.GetDocument).WhereNotNull());
+            return await RunServiceAsync(async () =>
+            {
+                // NOTE: In projection scenarios, we might get a set of documents to search
+                // that are not all the same language and might not exist in the OOP process
+                // (like the JS parts of a .cshtml file). Filter them out here.  This will
+                // need to be revisited if we someday support FAR between these languages.
+                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                var document = solution.GetDocument(documentId);
+                var documentsToSearch = ImmutableHashSet.CreateRange(
+                    documentIdsToSearch.Select(solution.GetDocument).WhereNotNull());
 
-            var service = document.GetLanguageService<IDocumentHighlightsService>();
-            var result = await service.GetDocumentHighlightsAsync(
-                document, position, documentsToSearch, cancellationToken).ConfigureAwait(false);
+                var service = document.GetLanguageService<IDocumentHighlightsService>();
+                var result = await service.GetDocumentHighlightsAsync(
+                    document, position, documentsToSearch, cancellationToken).ConfigureAwait(false);
 
-            return result.SelectAsArray(SerializableDocumentHighlights.Dehydrate);
+                return result.SelectAsArray(SerializableDocumentHighlights.Dehydrate);
+            }, cancellationToken).ConfigureAwait(false);
         }
     }
 }

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_NavigateTo.cs

@@ -13,31 +13,37 @@ internal partial class CodeAnalysisService : IRemoteNavigateToSearchService
         public async Task<IList<SerializableNavigateToSearchResult>> SearchDocumentAsync(
             DocumentId documentId, string searchPattern, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return await RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                using (UserOperationBooster.Boost())
+                {
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
 
-                var project = solution.GetDocument(documentId);
-                var result = await AbstractNavigateToSearchService.SearchDocumentInCurrentProcessAsync(
-                    project, searchPattern, cancellationToken).ConfigureAwait(false);
+                    var project = solution.GetDocument(documentId);
+                    var result = await AbstractNavigateToSearchService.SearchDocumentInCurrentProcessAsync(
+                        project, searchPattern, cancellationToken).ConfigureAwait(false);
 
-                return Convert(result);
-            }
+                    return Convert(result);
+                }
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         public async Task<IList<SerializableNavigateToSearchResult>> SearchProjectAsync(
             ProjectId projectId, string searchPattern, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return await RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                using (UserOperationBooster.Boost())
+                {
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
 
-                var project = solution.GetProject(projectId);
-                var result = await AbstractNavigateToSearchService.SearchProjectInCurrentProcessAsync(
-                    project, searchPattern, cancellationToken).ConfigureAwait(false);
+                    var project = solution.GetProject(projectId);
+                    var result = await AbstractNavigateToSearchService.SearchProjectInCurrentProcessAsync(
+                        project, searchPattern, cancellationToken).ConfigureAwait(false);
 
-                return Convert(result);
-            }
+                    return Convert(result);
+                }
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         private ImmutableArray<SerializableNavigateToSearchResult> Convert(

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_SymbolFinder.cs

@@ -15,110 +15,128 @@ namespace Microsoft.CodeAnalysis.Remote
     // root level service for all Roslyn services
     internal partial class CodeAnalysisService : IRemoteSymbolFinder
     {
-        public async Task FindReferencesAsync(SerializableSymbolAndProjectId symbolAndProjectIdArg, DocumentId[] documentArgs, CancellationToken cancellationToken)
+        public Task FindReferencesAsync(SerializableSymbolAndProjectId symbolAndProjectIdArg, DocumentId[] documentArgs, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
-
-                var symbolAndProjectId = await symbolAndProjectIdArg.TryRehydrateAsync(
-                    solution, cancellationToken).ConfigureAwait(false);
-
-                var progressCallback = new FindReferencesProgressCallback(this);
-
-                if (!symbolAndProjectId.HasValue)
+                using (UserOperationBooster.Boost())
                 {
-                    await progressCallback.OnStartedAsync().ConfigureAwait(false);
-                    await progressCallback.OnCompletedAsync().ConfigureAwait(false);
-                    return;
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+
+                    var symbolAndProjectId = await symbolAndProjectIdArg.TryRehydrateAsync(
+                        solution, cancellationToken).ConfigureAwait(false);
+
+                    var progressCallback = new FindReferencesProgressCallback(this);
+
+                    if (!symbolAndProjectId.HasValue)
+                    {
+                        await progressCallback.OnStartedAsync().ConfigureAwait(false);
+                        await progressCallback.OnCompletedAsync().ConfigureAwait(false);
+                        return;
+                    }
+
+                    // NOTE: In projection scenarios, we might get a set of documents to search
+                    // that are not all the same language and might not exist in the OOP process
+                    // (like the JS parts of a .cshtml file). Filter them out here.  This will
+                    // need to be revisited if we someday support FAR between these languages.
+                    var documents = documentArgs?.Select(solution.GetDocument)
+                                                 .WhereNotNull()
+                                                 .ToImmutableHashSet();
+
+                    await SymbolFinder.FindReferencesInCurrentProcessAsync(
+                        symbolAndProjectId.Value, solution,
+                        progressCallback, documents, cancellationToken).ConfigureAwait(false);
                 }
-
-                // NOTE: In projection scenarios, we might get a set of documents to search
-                // that are not all the same language and might not exist in the OOP process
-                // (like the JS parts of a .cshtml file). Filter them out here.  This will
-                // need to be revisited if we someday support FAR between these languages.
-                var documents = documentArgs?.Select(solution.GetDocument)
-                                             .WhereNotNull()
-                                             .ToImmutableHashSet();
-
-                await SymbolFinder.FindReferencesInCurrentProcessAsync(
-                    symbolAndProjectId.Value, solution,
-                    progressCallback, documents, cancellationToken).ConfigureAwait(false);
-            }
+            }, cancellationToken);
         }
 
-        public async Task FindLiteralReferencesAsync(object value, TypeCode typeCode, CancellationToken cancellationToken)
+        public Task FindLiteralReferencesAsync(object value, TypeCode typeCode, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return RunServiceAsync(async () =>
             {
-                var convertedType = System.Convert.ChangeType(value, typeCode);
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                using (UserOperationBooster.Boost())
+                {
+                    var convertedType = System.Convert.ChangeType(value, typeCode);
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
 
-                var progressCallback = new FindLiteralReferencesProgressCallback(this);
-                await SymbolFinder.FindLiteralReferencesInCurrentProcessAsync(
-                    convertedType, solution, progressCallback, cancellationToken).ConfigureAwait(false);
-            }
+                    var progressCallback = new FindLiteralReferencesProgressCallback(this);
+                    await SymbolFinder.FindLiteralReferencesInCurrentProcessAsync(
+                        convertedType, solution, progressCallback, cancellationToken).ConfigureAwait(false);
+                }
+            }, cancellationToken);
         }
 
         public async Task<IList<SerializableSymbolAndProjectId>> FindAllDeclarationsWithNormalQueryAsync(
             ProjectId projectId, string name, SearchKind searchKind, SymbolFilter criteria, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return await RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
-                var project = solution.GetProject(projectId);
-
-                using (var query = SearchQuery.Create(name, searchKind))
+                using (UserOperationBooster.Boost())
                 {
-                    var result = await DeclarationFinder.FindAllDeclarationsWithNormalQueryInCurrentProcessAsync(
-                        project, query, criteria, cancellationToken).ConfigureAwait(false);
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                    var project = solution.GetProject(projectId);
 
-                    return result.SelectAsArray(SerializableSymbolAndProjectId.Dehydrate);
+                    using (var query = SearchQuery.Create(name, searchKind))
+                    {
+                        var result = await DeclarationFinder.FindAllDeclarationsWithNormalQueryInCurrentProcessAsync(
+                            project, query, criteria, cancellationToken).ConfigureAwait(false);
+
+                        return result.SelectAsArray(SerializableSymbolAndProjectId.Dehydrate);
+                    }
                 }
-            }
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         public async Task<IList<SerializableSymbolAndProjectId>> FindSolutionSourceDeclarationsWithNormalQueryAsync(
             string name, bool ignoreCase, SymbolFilter criteria, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return await RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
-                var result = await DeclarationFinder.FindSourceDeclarationsWithNormalQueryInCurrentProcessAsync(
-                    solution, name, ignoreCase, criteria, cancellationToken).ConfigureAwait(false);
+                using (UserOperationBooster.Boost())
+                {
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                    var result = await DeclarationFinder.FindSourceDeclarationsWithNormalQueryInCurrentProcessAsync(
+                        solution, name, ignoreCase, criteria, cancellationToken).ConfigureAwait(false);
 
-                return result.SelectAsArray(SerializableSymbolAndProjectId.Dehydrate);
-            }
+                    return result.SelectAsArray(SerializableSymbolAndProjectId.Dehydrate);
+                }
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         public async Task<IList<SerializableSymbolAndProjectId>> FindProjectSourceDeclarationsWithNormalQueryAsync(
             ProjectId projectId, string name, bool ignoreCase, SymbolFilter criteria, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return await RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
-                var project = solution.GetProject(projectId);
+                using (UserOperationBooster.Boost())
+                {
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                    var project = solution.GetProject(projectId);
 
-                var result = await DeclarationFinder.FindSourceDeclarationsWithNormalQueryInCurrentProcessAsync(
-                    project, name, ignoreCase, criteria, cancellationToken).ConfigureAwait(false);
+                    var result = await DeclarationFinder.FindSourceDeclarationsWithNormalQueryInCurrentProcessAsync(
+                        project, name, ignoreCase, criteria, cancellationToken).ConfigureAwait(false);
 
-                return result.SelectAsArray(SerializableSymbolAndProjectId.Dehydrate);
-            }
+                    return result.SelectAsArray(SerializableSymbolAndProjectId.Dehydrate);
+                }
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         public async Task<IList<SerializableSymbolAndProjectId>> FindProjectSourceDeclarationsWithPatternAsync(
             ProjectId projectId, string pattern, SymbolFilter criteria, CancellationToken cancellationToken)
         {
-            using (UserOperationBooster.Boost())
+            return await RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
-                var project = solution.GetProject(projectId);
+                using (UserOperationBooster.Boost())
+                {
+                    var solution = await GetSolutionAsync(cancellationToken).ConfigureAwait(false);
+                    var project = solution.GetProject(projectId);
 
-                var result = await DeclarationFinder.FindSourceDeclarationsWithPatternInCurrentProcessAsync(
-                    project, pattern, criteria, cancellationToken).ConfigureAwait(false);
+                    var result = await DeclarationFinder.FindSourceDeclarationsWithPatternInCurrentProcessAsync(
+                        project, pattern, criteria, cancellationToken).ConfigureAwait(false);
 
-                return result.SelectAsArray(SerializableSymbolAndProjectId.Dehydrate);
-            }
+                    return result.SelectAsArray(SerializableSymbolAndProjectId.Dehydrate);
+                }
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         private class FindLiteralReferencesProgressCallback : IStreamingFindLiteralReferencesProgress

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_TodoComments.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Diagnostics;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
@@ -11,6 +12,7 @@
 using Microsoft.CodeAnalysis.Shared.Extensions;
 using Microsoft.CodeAnalysis.TodoComments;
 using Roslyn.Utilities;
+using StreamJsonRpc;
 using RoslynLogger = Microsoft.CodeAnalysis.Internal.Log.Logger;
 
 namespace Microsoft.CodeAnalysis.Remote
@@ -25,20 +27,23 @@ internal partial class CodeAnalysisService : IRemoteTodoCommentService
         /// </summary>
         public async Task<IList<TodoComment>> GetTodoCommentsAsync(PinnedSolutionInfo solutionInfo, DocumentId documentId, IList<TodoCommentDescriptor> tokens, CancellationToken cancellationToken)
         {
-            using (RoslynLogger.LogBlock(FunctionId.CodeAnalysisService_GetTodoCommentsAsync, documentId.ProjectId.DebugName, cancellationToken))
+            return await RunServiceAsync(async () =>
             {
-                var solution = await GetSolutionAsync(solutionInfo, cancellationToken).ConfigureAwait(false);
-                var document = solution.GetDocument(documentId);
-
-                var service = document.GetLanguageService<ITodoCommentService>();
-                if (service != null)
+                using (RoslynLogger.LogBlock(FunctionId.CodeAnalysisService_GetTodoCommentsAsync, documentId.ProjectId.DebugName, cancellationToken))
                 {
-                    // todo comment service supported
-                    return await service.GetTodoCommentsAsync(document, tokens, cancellationToken).ConfigureAwait(false);
-                }
+                    var solution = await GetSolutionAsync(solutionInfo, cancellationToken).ConfigureAwait(false);
+                    var document = solution.GetDocument(documentId);
 
-                return SpecializedCollections.EmptyList<TodoComment>();
-            }
+                    var service = document.GetLanguageService<ITodoCommentService>();
+                    if (service != null)
+                    {
+                        // todo comment service supported
+                        return await service.GetTodoCommentsAsync(document, tokens, cancellationToken).ConfigureAwait(false);
+                    }
+
+                    return SpecializedCollections.EmptyList<TodoComment>();
+                }
+            }, cancellationToken).ConfigureAwait(false);
         }
     }
 }

src/Workspaces/Remote/ServiceHub/Services/RemoteHostService.cs

@@ -5,7 +5,6 @@
 using System.ComponentModel;
 using System.Diagnostics;
 using System.IO;
-using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
@@ -15,7 +14,6 @@
 using Microsoft.CodeAnalysis.Notification;
 using Microsoft.CodeAnalysis.Remote.Services;
 using Microsoft.CodeAnalysis.Remote.Storage;
-using Microsoft.CodeAnalysis.Remote.Telemetry;
 using Microsoft.CodeAnalysis.Storage;
 using Microsoft.VisualStudio.LanguageServices.Telemetry;
 using Microsoft.VisualStudio.Telemetry;
@@ -60,68 +58,89 @@ static RemoteHostService()
 
         public string Connect(string host, string serializedSession)
         {
-            _primaryInstance = InstanceId;
+            return RunService(() =>
+            {
+                _primaryInstance = InstanceId;
 
-            var existing = Interlocked.CompareExchange(ref _host, host, null);
+                var existing = Interlocked.CompareExchange(ref _host, host, null);
 
-            SetGlobalContext(serializedSession);
+                SetGlobalContext(serializedSession);
 
-            if (existing != null && existing != host)
-            {
-                LogError($"{host} is given for {existing}");
-            }
+                if (existing != null && existing != host)
+                {
+                    LogError($"{host} is given for {existing}");
+                }
 
-            // log telemetry that service hub started
-            RoslynLogger.Log(FunctionId.RemoteHost_Connect, KeyValueLogMessage.Create(SetSessionInfo));
+                // log telemetry that service hub started
+                RoslynLogger.Log(FunctionId.RemoteHost_Connect, KeyValueLogMessage.Create(SetSessionInfo));
 
-            return _host;
+                return _host;
+            }, CancellationToken.None);
         }
 
-        public async Task SynchronizePrimaryWorkspaceAsync(Checksum checksum, CancellationToken cancellationToken)
+        public Task SynchronizePrimaryWorkspaceAsync(Checksum checksum, CancellationToken cancellationToken)
         {
-            using (RoslynLogger.LogBlock(FunctionId.RemoteHostService_SynchronizePrimaryWorkspaceAsync, Checksum.GetChecksumLogInfo, checksum, cancellationToken))
+            return RunServiceAsync(async () =>
             {
-                var solutionController = (ISolutionController)RoslynServices.SolutionService;
-                await solutionController.UpdatePrimaryWorkspaceAsync(checksum, cancellationToken).ConfigureAwait(false);
-            }
+                using (RoslynLogger.LogBlock(FunctionId.RemoteHostService_SynchronizePrimaryWorkspaceAsync, Checksum.GetChecksumLogInfo, checksum, cancellationToken))
+                {
+                    var solutionController = (ISolutionController)RoslynServices.SolutionService;
+                    await solutionController.UpdatePrimaryWorkspaceAsync(checksum, cancellationToken).ConfigureAwait(false);
+                }
+            }, cancellationToken);
         }
 
-        public async Task SynchronizeGlobalAssetsAsync(Checksum[] checksums, CancellationToken cancellationToken)
+        public Task SynchronizeGlobalAssetsAsync(Checksum[] checksums, CancellationToken cancellationToken)
         {
-            using (RoslynLogger.LogBlock(FunctionId.RemoteHostService_SynchronizeGlobalAssetsAsync, Checksum.GetChecksumsLogInfo, checksums, cancellationToken))
+            return RunServiceAsync(async () =>
             {
-                var assets = await RoslynServices.AssetService.GetAssetsAsync<object>(checksums, cancellationToken).ConfigureAwait(false);
-
-                foreach (var asset in assets)
+                using (RoslynLogger.LogBlock(FunctionId.RemoteHostService_SynchronizeGlobalAssetsAsync, Checksum.GetChecksumsLogInfo, checksums, cancellationToken))
                 {
-                    AssetStorage.TryAddGlobalAsset(asset.Item1, asset.Item2);
+                    var assets = await RoslynServices.AssetService.GetAssetsAsync<object>(checksums, cancellationToken).ConfigureAwait(false);
+
+                    foreach (var asset in assets)
+                    {
+                        AssetStorage.TryAddGlobalAsset(asset.Item1, asset.Item2);
+                    }
                 }
-            }
+            }, cancellationToken);
         }
 
         public void RegisterPrimarySolutionId(SolutionId solutionId, string storageLocation, CancellationToken cancellationToken)
         {
-            var persistentStorageService = GetPersistentStorageService();
-            persistentStorageService?.RegisterPrimarySolution(solutionId);
-            RemotePersistentStorageLocationService.UpdateStorageLocation(solutionId, storageLocation);
+            RunService(() =>
+            {
+                var persistentStorageService = GetPersistentStorageService();
+                persistentStorageService?.RegisterPrimarySolution(solutionId);
+                RemotePersistentStorageLocationService.UpdateStorageLocation(solutionId, storageLocation);
+            }, cancellationToken);
         }
 
         public void UnregisterPrimarySolutionId(SolutionId solutionId, bool synchronousShutdown, CancellationToken cancellationToken)
         {
-            var persistentStorageService = GetPersistentStorageService();
-            persistentStorageService?.UnregisterPrimarySolution(solutionId, synchronousShutdown);
+            RunService(() =>
+            {
+                var persistentStorageService = GetPersistentStorageService();
+                persistentStorageService?.UnregisterPrimarySolution(solutionId, synchronousShutdown);
+            }, cancellationToken);
         }
 
         public void OnGlobalOperationStarted(string unused)
         {
-            var globalOperationNotificationService = GetGlobalOperationNotificationService();
-            globalOperationNotificationService?.OnStarted();
+            RunService(() =>
+            {
+                var globalOperationNotificationService = GetGlobalOperationNotificationService();
+                globalOperationNotificationService?.OnStarted();
+            }, CancellationToken.None);
         }
 
         public void OnGlobalOperationStopped(IReadOnlyList<string> operations, bool cancelled)
         {
-            var globalOperationNotificationService = GetGlobalOperationNotificationService();
-            globalOperationNotificationService?.OnStopped(operations, cancelled);
+            RunService(() =>
+            {
+                var globalOperationNotificationService = GetGlobalOperationNotificationService();
+                globalOperationNotificationService?.OnStopped(operations, cancelled);
+            }, CancellationToken.None);
         }
 
         private static Func<FunctionId, bool> GetLoggingChecker()
@@ -208,7 +227,7 @@ private static AbstractPersistentStorageService GetPersistentStorageService()
             var persistentStorageService = workspace.Services.GetService<IPersistentStorageService>() as AbstractPersistentStorageService;
             return persistentStorageService;
         }
-        
+
         private RemoteGlobalOperationNotificationService GetGlobalOperationNotificationService()
         {
             var workspace = SolutionService.PrimaryWorkspace;

src/Workspaces/Remote/ServiceHub/Services/RemoteSymbolSearchUpdateEngine.cs

@@ -10,7 +10,7 @@
 
 namespace Microsoft.CodeAnalysis.Remote
 {
-    internal partial class RemoteSymbolSearchUpdateEngine : 
+    internal partial class RemoteSymbolSearchUpdateEngine :
         ServiceHubServiceBase, IRemoteSymbolSearchUpdateEngine, ISymbolSearchLogService, ISymbolSearchProgressService
     {
         private readonly SymbolSearchUpdateEngine _updateEngine;
@@ -26,31 +26,43 @@ public RemoteSymbolSearchUpdateEngine(Stream stream, IServiceProvider servicePro
 
         public Task UpdateContinuouslyAsync(string sourceName, string localSettingsDirectory)
         {
-            return _updateEngine.UpdateContinuouslyAsync(sourceName, localSettingsDirectory);
+            return RunServiceAsync(() =>
+            {
+                return _updateEngine.UpdateContinuouslyAsync(sourceName, localSettingsDirectory);
+            }, CancellationToken.None);
         }
 
         public async Task<IList<PackageWithTypeResult>> FindPackagesWithTypeAsync(string source, string name, int arity, CancellationToken cancellationToken)
         {
-            var results = await _updateEngine.FindPackagesWithTypeAsync(
-                source, name, arity, cancellationToken).ConfigureAwait(false);
+            return await RunServiceAsync(async () =>
+            {
+                var results = await _updateEngine.FindPackagesWithTypeAsync(
+                    source, name, arity, cancellationToken).ConfigureAwait(false);
 
-            return results;
+                return results;
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         public async Task<IList<PackageWithAssemblyResult>> FindPackagesWithAssemblyAsync(string source, string assemblyName, CancellationToken cancellationToken)
         {
-            var results = await _updateEngine.FindPackagesWithAssemblyAsync(
-                source, assemblyName, cancellationToken).ConfigureAwait(false);
+            return await RunServiceAsync(async () =>
+            {
+                var results = await _updateEngine.FindPackagesWithAssemblyAsync(
+                    source, assemblyName, cancellationToken).ConfigureAwait(false);
 
-            return results;
+                return results;
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         public async Task<IList<ReferenceAssemblyWithTypeResult>> FindReferenceAssembliesWithTypeAsync(string name, int arity, CancellationToken cancellationToken)
         {
-            var results = await _updateEngine.FindReferenceAssembliesWithTypeAsync(
-                name, arity, cancellationToken).ConfigureAwait(false);
+            return await RunServiceAsync(async () =>
+            {
+                var results = await _updateEngine.FindReferenceAssembliesWithTypeAsync(
+                    name, arity, cancellationToken).ConfigureAwait(false);
 
-            return results;
+                return results;
+            }, cancellationToken).ConfigureAwait(false);
         }
 
         #region Messages to forward from here to VS

src/Workspaces/Remote/ServiceHub/Services/SnapshotService.JsonRpcAssetSource.cs

@@ -34,10 +34,32 @@ public override async Task<IList<(Checksum, object)>> RequestAssetsAsync(int sco
             {
                 using (RoslynLogger.LogBlock(FunctionId.SnapshotService_RequestAssetAsync, GetRequestLogInfo, scopeId, checksums, cancellationToken))
                 {
-                    return await _owner.Rpc.InvokeAsync(WellKnownServiceHubServices.AssetService_RequestAssetAsync,
-                        new object[] { scopeId, checksums.ToArray() },
-                        (s, c) => ReadAssets(s, scopeId, checksums, c), cancellationToken).ConfigureAwait(false);
+                    try
+                    {
+                        return await _owner.RunServiceAsync(() =>
+                        {
+                            return _owner.Rpc.InvokeAsync(WellKnownServiceHubServices.AssetService_RequestAssetAsync,
+                                new object[] { scopeId, checksums.ToArray() },
+                                (s, c) => ReadAssets(s, scopeId, checksums, c), cancellationToken);
+                        }, cancellationToken).ConfigureAwait(false);
+                    }
+                    catch (Exception ex) when (ReportUnlessCanceled(ex, cancellationToken))
+                    {
+                        throw ExceptionUtilities.Unreachable;
+                    }
+                }
+            }
+
+            private bool ReportUnlessCanceled(Exception ex, CancellationToken cancellationToken)
+            {
+                if (!cancellationToken.IsCancellationRequested &&
+                    ((IDisposableObservable)_owner.Rpc).IsDisposed)
+                {
+                    // kill OOP if snapshot service got disconnected due to this exception.
+                    FailFast.OnFatalException(ex);
                 }
+
+                return false;
             }
 
             private IList<(Checksum, object)> ReadAssets(

src/Workspaces/Remote/ServiceHub/Shared/Extensions.cs

@@ -7,6 +7,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.ErrorReporting;
+using Microsoft.VisualStudio.Telemetry;
 using Roslyn.Utilities;
 using StreamJsonRpc;
 
@@ -18,13 +19,15 @@ internal static partial class Extensions
             this JsonRpc rpc, string targetName, IReadOnlyList<object> arguments,
             Func<Stream, CancellationToken, Task> funcWithDirectStreamAsync, CancellationToken cancellationToken)
         {
+            Task task = null;
+
             using (var mergedCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
             using (var stream = new ServerDirectStream())
             {
                 try
                 {
                     // send request by adding direct stream name to end of arguments
-                    var task = rpc.InvokeWithCancellationAsync(targetName, arguments.Concat(stream.Name).ToArray(), mergedCancellation.Token);
+                    task = rpc.InvokeWithCancellationAsync(targetName, arguments.Concat(stream.Name).ToArray(), cancellationToken);
 
                     // if invoke throws an exception, make sure we raise cancellation.
                     RaiseCancellationIfInvokeFailed(task, mergedCancellation, cancellationToken);
@@ -47,6 +50,11 @@ internal static partial class Extensions
                     // but we need merged one to cancel operation if InvokeAsync has failed. if it failed without
                     // cancellation token is raised, then we do want to have watson report
                     cancellationToken.ThrowIfCancellationRequested();
+
+                    // record reason why task got aborted. use NFW here since we don't want to
+                    // crash VS on explicitly killing OOP.
+                    task.Exception.ReportServiceHubNFW("JsonRpc Invoke Failed");
+
                     throw;
                 }
             }
@@ -56,13 +64,15 @@ internal static partial class Extensions
             this JsonRpc rpc, string targetName, IReadOnlyList<object> arguments,
             Func<Stream, CancellationToken, Task<T>> funcWithDirectStreamAsync, CancellationToken cancellationToken)
         {
+            Task task = null;
+
             using (var mergedCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
             using (var stream = new ServerDirectStream())
             {
                 try
                 {
                     // send request to asset source
-                    var task = rpc.InvokeWithCancellationAsync(targetName, arguments.Concat(stream.Name).ToArray(), mergedCancellation.Token);
+                    task = rpc.InvokeWithCancellationAsync(targetName, arguments.Concat(stream.Name).ToArray(), cancellationToken);
 
                     // if invoke throws an exception, make sure we raise cancellation.
                     RaiseCancellationIfInvokeFailed(task, mergedCancellation, cancellationToken);
@@ -85,6 +95,11 @@ internal static partial class Extensions
                     // is raised, it can cause one to be in cancelled mode and the other is not. here, one we
                     // actually care is the cancellation token given in, not the merged cancellation token.
                     cancellationToken.ThrowIfCancellationRequested();
+
+                    // record reason why task got aborted. use NFW here since we don't want to
+                    // crash VS on explicitly killing OOP.
+                    task.Exception.ReportServiceHubNFW("JsonRpc Invoke Failed");
+
                     throw;
                 }
             }
@@ -94,13 +109,15 @@ internal static partial class Extensions
             this JsonRpc rpc, string targetName, IReadOnlyList<object> arguments,
             Action<Stream, CancellationToken> actionWithDirectStream, CancellationToken cancellationToken)
         {
+            Task task = null;
+
             using (var mergedCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
             using (var stream = new ServerDirectStream())
             {
                 try
                 {
                     // send request by adding direct stream name to end of arguments
-                    var task = rpc.InvokeWithCancellationAsync(targetName, arguments.Concat(stream.Name).ToArray(), mergedCancellation.Token);
+                    task = rpc.InvokeWithCancellationAsync(targetName, arguments.Concat(stream.Name).ToArray(), cancellationToken);
 
                     // if invoke throws an exception, make sure we raise cancellation.
                     RaiseCancellationIfInvokeFailed(task, mergedCancellation, cancellationToken);
@@ -121,6 +138,11 @@ internal static partial class Extensions
                     // is raised, it can cause one to be in cancelled mode and the other is not. here, one we
                     // actually care is the cancellation token given in, not the merged cancellation token.
                     cancellationToken.ThrowIfCancellationRequested();
+
+                    // record reason why task got aborted. use NFW here since we don't want to
+                    // crash VS on explicitly killing OOP.
+                    task.Exception.ReportServiceHubNFW("JsonRpc Invoke Failed");
+
                     throw;
                 }
             }
@@ -130,13 +152,15 @@ internal static partial class Extensions
             this JsonRpc rpc, string targetName, IReadOnlyList<object> arguments,
             Func<Stream, CancellationToken, T> funcWithDirectStream, CancellationToken cancellationToken)
         {
+            Task task = null;
+
             using (var mergedCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
             using (var stream = new ServerDirectStream())
             {
                 try
                 {
                     // send request to asset source
-                    var task = rpc.InvokeWithCancellationAsync(targetName, arguments.Concat(stream.Name).ToArray(), mergedCancellation.Token);
+                    task = rpc.InvokeWithCancellationAsync(targetName, arguments.Concat(stream.Name).ToArray(), cancellationToken);
 
                     // if invoke throws an exception, make sure we raise cancellation.
                     RaiseCancellationIfInvokeFailed(task, mergedCancellation, cancellationToken);
@@ -159,6 +183,11 @@ internal static partial class Extensions
                     // is raised, it can cause one to be in cancelled mode and the other is not. here, one we
                     // actually care is the cancellation token given in, not the merged cancellation token.
                     cancellationToken.ThrowIfCancellationRequested();
+
+                    // record reason why task got aborted. use NFW here since we don't want to
+                    // crash VS on explicitly killing OOP.
+                    task.Exception.ReportServiceHubNFW("JsonRpc Invoke Failed");
+
                     throw;
                 }
             }
@@ -214,5 +243,52 @@ private static void RaiseCancellationIfInvokeFailed(Task task, CancellationToken
                 }
             }, cancellationToken, TaskContinuationOptions.NotOnRanToCompletion | TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
         }
+
+        public static void ReportServiceHubNFW(this Exception exception, string message)
+        {
+            if (exception == null)
+            {
+                return;
+            }
+
+            WatsonReporter.Report(message, exception, ReportDetailServiceHubLogs);
+        }
+
+        private static int ReportDetailServiceHubLogs(IFaultUtility faultUtility)
+        {
+            // 0 means send watson, otherwise, cancel watson
+            // we always send watson since dump itself can have valuable data
+            var exitCode = 0;
+
+            try
+            {
+                var logPath = Path.Combine(Path.GetTempPath(), "servicehub", "logs");
+                if (!Directory.Exists(logPath))
+                {
+                    return exitCode;
+                }
+
+                // attach all log files that are modified less than 1 day before.
+                var now = DateTime.UtcNow;
+                var oneDay = TimeSpan.FromDays(1);
+
+                foreach (var file in Directory.EnumerateFiles(logPath, "*.log"))
+                {
+                    var lastWrite = File.GetLastWriteTimeUtc(file);
+                    if (now - lastWrite > oneDay)
+                    {
+                        continue;
+                    }
+
+                    faultUtility.AddFile(file);
+                }
+            }
+            catch (Exception)
+            {
+                // it is okay to fail on reporting watson
+            }
+
+            return exitCode;
+        }
     }
 }

src/Workspaces/Remote/ServiceHub/Shared/ServiceHubServiceBase.cs

@@ -169,5 +169,69 @@ private static Task<Solution> GetSolutionAsync(RoslynServices roslynService, Pin
             var solutionController = (ISolutionController)roslynService.SolutionService;
             return solutionController.GetSolutionAsync(solutionInfo.SolutionChecksum, solutionInfo.FromPrimaryBranch, cancellationToken);
         }
+
+        protected async Task<T> RunServiceAsync<T>(Func<Task<T>> callAsync, CancellationToken cancellationToken)
+        {
+            try
+            {
+                return await callAsync().ConfigureAwait(false);
+            }
+            catch (Exception ex) when (LogUnlessCanceled(ex, cancellationToken))
+            {
+                // never reach
+                return default(T);
+            }
+        }
+
+        protected async Task RunServiceAsync(Func<Task> callAsync, CancellationToken cancellationToken)
+        {
+            try
+            {
+                await callAsync().ConfigureAwait(false);
+            }
+            catch (Exception ex) when (LogUnlessCanceled(ex, cancellationToken))
+            {
+                // never reach
+                return;
+            }
+        }
+
+        protected T RunService<T>(Func<T> call, CancellationToken cancellationToken)
+        {
+            try
+            {
+                return call();
+            }
+            catch (Exception ex) when (LogUnlessCanceled(ex, cancellationToken))
+            {
+                // never reach
+                return default;
+            }
+        }
+
+        protected void RunService(Action call, CancellationToken cancellationToken)
+        {
+            try
+            {
+                call();
+            }
+            catch (Exception ex) when (LogUnlessCanceled(ex, cancellationToken))
+            {
+                // never reach
+            }
+        }
+
+        private bool LogUnlessCanceled(Exception ex, CancellationToken cancellationToken)
+        {
+            if (!cancellationToken.IsCancellationRequested)
+            {
+                LogError("Exception: " + ex.ToString());
+
+                var callStack = new StackTrace().ToString();
+                LogError("From: " + callStack);
+            }
+
+            return false;
+        }
     }
 }

src/Workspaces/Remote/ServiceHub/Telemetry/WatsonReporter.cs

@@ -5,12 +5,18 @@
 using Microsoft.VisualStudio.LanguageServices.Telemetry;
 using Microsoft.VisualStudio.Telemetry;
 
-namespace Microsoft.CodeAnalysis.Remote.Telemetry
+namespace Microsoft.CodeAnalysis.ErrorReporting
 {
     internal class WatsonReporter
     {
         private static TelemetrySession s_sessionOpt;
 
+        /// <summary>
+        /// The default callback to pass to <see cref="TelemetrySessionExtensions.PostFault(TelemetrySession, string, string, Exception, Func{IFaultUtility, int})"/>.
+        /// Returning "0" signals that we should send data to Watson; any other value will cancel the Watson report.
+        /// </summary>
+        private static Func<IFaultUtility, int> s_defaultCallback = _ => 0;
+
         /// <summary>
         /// Set default telemetry session
         /// </summary>
@@ -39,6 +45,19 @@ public static void Report(Exception exception)
         /// <param name="description">any description you want to save with this watson report</param>
         /// <param name="exception">Exception that triggered this non-fatal error</param>
         public static void Report(string description, Exception exception)
+        {
+            Report(description, exception, s_defaultCallback);
+        }
+
+        /// <summary>
+        /// Report Non-Fatal Watson
+        /// </summary>
+        /// <param name="description">any description you want to save with this watson report</param>
+        /// <param name="exception">Exception that triggered this non-fatal error</param>
+        /// <param name="callback">Callback to include extra data with the NFW. Note that we always collect
+        /// a dump of the current process, but this can be used to add further information or files to the
+        /// CAB.</param>
+        public static void Report(string description, Exception exception, Func<IFaultUtility, int> callback)
         {
             // if given exception is non recoverable exception,
             // crash instead of NFW
@@ -53,11 +72,9 @@ public static void Report(string description, Exception exception)
                 exceptionObject: exception,
                 gatherEventDetails: arg =>
                 {
+                    // always add current processes dump
                     arg.AddProcessDump(System.Diagnostics.Process.GetCurrentProcess().Id);
-
-                    // 0 means send watson, otherwise, cancel watson
-                    // we always send watson since dump itself can have valuable data
-                    return 0;
+                    return callback(arg);
                 });
         }