Blame (#24043)

* enable logAnalyzerExecutionTime on IDE so that we can start track analyzer perf

* removed unnecessary ICodeAnalysisDiagnosticAnalyzerExecutor interface.

it was added when OOP is first introduced to make sure VS.Next dll won't get loaded to VS process if OOP is not enabled.

when it is enabled by default, rather than removing the interface, implementation just moved down to feature layer to reduce code churn.

now, I am properly removing unnecessary abstraction.

* take Executor out of test MEF composition

* added IRemoteDiagnosticAnalyzerService interface

* made initial version working.

* added tests

* add tracking for inproc only analyzers

* pass in diagnostic analyzer service

* added open file performance tracking as well.

* added PII test

* dont hash analyzerId when it is reported by internal user

* added link to LOF wiki

* made blame to track open files as well.

* forgot to add return in if statement

* reduce threshold to 100ms

decide to start from lower threshold and then iterate rather than start from higher threshold

* added a way to log real time perf data in local machine with explicit option which can be used to train formula later

* addressed ivan's feedbacks

* renamed to ExpensiveAnalyzerInfo

* addressed PR feedbacks

* more renames

* addressed PR feedbacks.

renamed as much as I can find.

* listener can be null in unit test

* addressed PR feedbacks

src/EditorFeatures/TestUtilities/ServiceTestExportProvider.cs

@@ -34,8 +34,7 @@ public static Type[] GetLanguageNeutralTypes()
                 typeof(Microsoft.CodeAnalysis.Editor.Implementation.SmartIndent.SmartIndentProvider),
                 typeof(Microsoft.CodeAnalysis.Editor.Implementation.ForegroundNotification.ForegroundNotificationService),
                 typeof(Implementation.InlineRename.InlineRenameService), // Ensure that EditorFeatures.Wpf is included in the composition
-                typeof(IncrementalCaches.SymbolTreeInfoIncrementalAnalyzerProvider),
-                typeof(CodeAnalysis.Diagnostics.EngineV2.DiagnosticAnalyzerExecutor)
+                typeof(IncrementalCaches.SymbolTreeInfoIncrementalAnalyzerProvider)
             };
 
             return MinimalTestExportProvider.GetLanguageNeutralTypes().Concat(types).Distinct().ToArray();

src/Features/Core/Portable/Diagnostics/AnalyzerHelper.cs

@@ -10,6 +10,9 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Collections.Generic;
+using Microsoft.CodeAnalysis.Diagnostics.Log;
+using Microsoft.CodeAnalysis.Diagnostics.Telemetry;
+using System.Collections.Immutable;
 
 namespace Microsoft.CodeAnalysis.Diagnostics
 {
@@ -294,5 +297,15 @@ public static void AppendAnalyzerMap(this Dictionary<string, DiagnosticAnalyzer>
                 analyzerMap[analyzer.GetAnalyzerId()] = analyzer;
             }
         }
+
+        public static IEnumerable<AnalyzerPerformanceInfo> ToAnalyzerPerformanceInfo(this IDictionary<DiagnosticAnalyzer, AnalyzerTelemetryInfo> analysisResult, IDiagnosticAnalyzerService serviceOpt = null)
+        {
+            return Convert(analysisResult.Select(kv => (kv.Key, kv.Value.ExecutionTime)), serviceOpt);
+        }
+
+        private static IEnumerable<AnalyzerPerformanceInfo> Convert(IEnumerable<(DiagnosticAnalyzer analyzer, TimeSpan timeSpan)> analyzerPerf, IDiagnosticAnalyzerService serviceOpt = null)
+        {
+            return analyzerPerf.Select(kv => new AnalyzerPerformanceInfo(kv.analyzer.GetAnalyzerId(), DiagnosticAnalyzerLogger.AllowsTelemetry(kv.analyzer, serviceOpt), kv.timeSpan));
+        }
     }
 }

src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer.CompilationManager.cs

@@ -82,7 +82,7 @@ public Task<CompilationWithAnalyzers> CreateAnalyzerDriverAsync(Project project,
 
                 // Create driver that holds onto compilation and associated analyzers
                 return CreateAnalyzerDriver(
-                    project, compilation, analyzers, logAnalyzerExecutionTime: false, reportSuppressedDiagnostics: includeSuppressedDiagnostics);
+                    project, compilation, analyzers, logAnalyzerExecutionTime: true, reportSuppressedDiagnostics: includeSuppressedDiagnostics);
             }
 
             private CompilationWithAnalyzers CreateAnalyzerDriver(

src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer.Executor.cs

@@ -9,7 +9,6 @@
 using Microsoft.CodeAnalysis.Diagnostics.Log;
 using Microsoft.CodeAnalysis.Diagnostics.Telemetry;
 using Microsoft.CodeAnalysis.ErrorReporting;
-using Microsoft.CodeAnalysis.Execution;
 using Microsoft.CodeAnalysis.Internal.Log;
 using Microsoft.CodeAnalysis.Shared.Options;
 using Microsoft.CodeAnalysis.Text;
@@ -27,10 +26,12 @@ internal partial class DiagnosticIncrementalAnalyzer
         private class Executor
         {
             private readonly DiagnosticIncrementalAnalyzer _owner;
+            private readonly InProcOrRemoteHostAnalyzerRunner _diagnosticAnalyzerRunner;
 
             public Executor(DiagnosticIncrementalAnalyzer owner)
             {
                 _owner = owner;
+                _diagnosticAnalyzerRunner = new InProcOrRemoteHostAnalyzerRunner(_owner.Owner, _owner.HostDiagnosticUpdateSource);
             }
 
             public IEnumerable<DiagnosticData> ConvertToLocalDiagnostics(Document targetDocument, IEnumerable<Diagnostic> diagnostics, TextSpan? span = null)
@@ -578,8 +579,7 @@ private IEnumerable<DiagnosticData> ConvertToLocalDiagnosticsWithoutCompilation(
                         ImmutableDictionary<DiagnosticAnalyzer, AnalyzerTelemetryInfo>.Empty);
                 }
 
-                var executor = project.Solution.Workspace.Services.GetService<ICodeAnalysisDiagnosticAnalyzerExecutor>();
-                return await executor.AnalyzeAsync(analyzerDriver, project, forcedAnalysis, cancellationToken).ConfigureAwait(false);
+                return await _diagnosticAnalyzerRunner.AnalyzeAsync(analyzerDriver, project, forcedAnalysis, cancellationToken).ConfigureAwait(false);
             }
 
             private IEnumerable<DiagnosticData> ConvertToLocalDiagnosticsWithCompilation(Document targetDocument, IEnumerable<Diagnostic> diagnostics, TextSpan? span = null)

src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticAnalyzerExecutor.cs → src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer.InProcOrRemoteHostAnalyzerRunner.cs

@@ -4,49 +4,36 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
-using System.Composition;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.Diagnostics.Telemetry;
+using Microsoft.CodeAnalysis.ErrorReporting;
 using Microsoft.CodeAnalysis.Execution;
-using Microsoft.CodeAnalysis.Host;
-using Microsoft.CodeAnalysis.Host.Mef;
 using Microsoft.CodeAnalysis.Options;
 using Microsoft.CodeAnalysis.Remote;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
 using Microsoft.CodeAnalysis.Workspaces.Diagnostics;
 using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis.Diagnostics.EngineV2
 {
-    [ExportWorkspaceServiceFactory(typeof(ICodeAnalysisDiagnosticAnalyzerExecutor)), Shared]
-    internal class DiagnosticAnalyzerExecutor : IWorkspaceServiceFactory
+    internal partial class DiagnosticIncrementalAnalyzer
     {
-        private readonly AbstractHostDiagnosticUpdateSource _hostDiagnosticUpdateSourceOpt;
-
-        [ImportingConstructor]
-        public DiagnosticAnalyzerExecutor([Import(AllowDefault = true)]AbstractHostDiagnosticUpdateSource hostDiagnosticUpdateSource)
-        {
-            // hostDiagnosticUpdateSource can be null in unit test
-            _hostDiagnosticUpdateSourceOpt = hostDiagnosticUpdateSource;
-        }
-
-        public IWorkspaceService CreateService(HostWorkspaceServices workspaceServices)
-        {
-            return new AnalyzerExecutor(_hostDiagnosticUpdateSourceOpt);
-        }
-
-        private class AnalyzerExecutor : ICodeAnalysisDiagnosticAnalyzerExecutor
+        // internal for testing
+        internal class InProcOrRemoteHostAnalyzerRunner
         {
+            private readonly DiagnosticAnalyzerService _owner;
             private readonly AbstractHostDiagnosticUpdateSource _hostDiagnosticUpdateSourceOpt;
 
             // TODO: this should be removed once we move options down to compiler layer
             private readonly ConcurrentDictionary<string, ValueTuple<OptionSet, CustomAsset>> _lastOptionSetPerLanguage;
 
-            public AnalyzerExecutor(AbstractHostDiagnosticUpdateSource hostDiagnosticUpdateSource)
+            public InProcOrRemoteHostAnalyzerRunner(DiagnosticAnalyzerService owner, AbstractHostDiagnosticUpdateSource hostDiagnosticUpdateSource)
             {
+                _owner = owner;
                 _hostDiagnosticUpdateSourceOpt = hostDiagnosticUpdateSource;
 
                 // currently option is a bit wierd since it is not part of snapshot and 
@@ -80,7 +67,7 @@ public AnalyzerExecutor(AbstractHostDiagnosticUpdateSource hostDiagnosticUpdateS
                 }
 
                 // due to OpenFileOnly analyzer, we need to run inproc as well for such analyzers
-                var inProcResultTask = AnalyzeInProcAsync(CreateAnalyzerDriver(analyzerDriver, a => a.IsOpenFileOnly(project.Solution.Workspace)), project, cancellationToken);
+                var inProcResultTask = AnalyzeInProcAsync(CreateAnalyzerDriver(analyzerDriver, a => a.IsOpenFileOnly(project.Solution.Workspace)), project, remoteHostClient, cancellationToken);
                 var outOfProcResultTask = AnalyzeOutOfProcAsync(remoteHostClient, analyzerDriver, project, forcedAnalysis, cancellationToken);
 
                 // run them concurrently in vs and remote host
@@ -95,8 +82,14 @@ public AnalyzerExecutor(AbstractHostDiagnosticUpdateSource hostDiagnosticUpdateS
                     inProcResultTask.Result.TelemetryInfo.AddRange(outOfProcResultTask.Result.TelemetryInfo));
             }
 
-            private async Task<DiagnosticAnalysisResultMap<DiagnosticAnalyzer, DiagnosticAnalysisResult>> AnalyzeInProcAsync(
+            private Task<DiagnosticAnalysisResultMap<DiagnosticAnalyzer, DiagnosticAnalysisResult>> AnalyzeInProcAsync(
                 CompilationWithAnalyzers analyzerDriver, Project project, CancellationToken cancellationToken)
+            {
+                return AnalyzeInProcAsync(analyzerDriver, project, client: null, cancellationToken);
+            }
+
+            private async Task<DiagnosticAnalysisResultMap<DiagnosticAnalyzer, DiagnosticAnalysisResult>> AnalyzeInProcAsync(
+                CompilationWithAnalyzers analyzerDriver, Project project, RemoteHostClient client, CancellationToken cancellationToken)
             {
                 if (analyzerDriver == null ||
                     analyzerDriver.Analyzers.Length == 0)
@@ -110,12 +103,41 @@ public AnalyzerExecutor(AbstractHostDiagnosticUpdateSource hostDiagnosticUpdateS
                 // PERF: Run all analyzers at once using the new GetAnalysisResultAsync API.
                 var analysisResult = await analyzerDriver.GetAnalysisResultAsync(cancellationToken).ConfigureAwait(false);
 
+                // if remote host is there, report performance data
+                var asyncToken = _owner?.Listener.BeginAsyncOperation(nameof(AnalyzeInProcAsync));
+                var _ = FireAndForgetReportAnalyzerPerformanceAsync(project, client, analysisResult, cancellationToken).CompletesAsyncOperation(asyncToken);
+
                 // get compiler result builder map
                 var builderMap = analysisResult.ToResultBuilderMap(project, version, analyzerDriver.Compilation, analyzerDriver.Analyzers, cancellationToken);
 
                 return DiagnosticAnalysisResultMap.Create(builderMap.ToImmutableDictionary(kv => kv.Key, kv => new DiagnosticAnalysisResult(kv.Value)), analysisResult.AnalyzerTelemetryInfo);
             }
 
+            private async Task FireAndForgetReportAnalyzerPerformanceAsync(Project project, RemoteHostClient client, AnalysisResult analysisResult, CancellationToken cancellationToken)
+            {
+                if (client == null)
+                {
+                    return;
+                }
+
+                try
+                {
+                    await client.TryRunCodeAnalysisRemoteAsync(
+                        nameof(IRemoteDiagnosticAnalyzerService.ReportAnalyzerPerformance),
+                        new object[]
+                        {
+                            analysisResult.AnalyzerTelemetryInfo.ToAnalyzerPerformanceInfo(_owner),
+                            // +1 for project itself
+                            project.DocumentIds.Count + 1
+                        },
+                        cancellationToken).ConfigureAwait(false);
+                }
+                catch (Exception ex) when (FatalError.ReportWithoutCrashUnlessCanceled(ex))
+                {
+                    // ignore all, this is fire and forget method
+                }
+            }
+
             private async Task<DiagnosticAnalysisResultMap<DiagnosticAnalyzer, DiagnosticAnalysisResult>> AnalyzeOutOfProcAsync(
                 RemoteHostClient client, CompilationWithAnalyzers analyzerDriver, Project project, bool forcedAnalysis, CancellationToken cancellationToken)
             {
@@ -149,7 +171,7 @@ public AnalyzerExecutor(AbstractHostDiagnosticUpdateSource hostDiagnosticUpdateS
                         session.AddAdditionalAssets(optionAsset);
 
                         var result = await session.InvokeAsync(
-                            WellKnownServiceHubServices.CodeAnalysisService_CalculateDiagnosticsAsync,
+                            nameof(IRemoteDiagnosticAnalyzerService.CalculateDiagnosticsAsync),
                             new object[] { argument },
                             (s, c) => GetCompilerAnalysisResultAsync(s, analyzerMap, project, c), cancellationToken).ConfigureAwait(false);
 

src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer_IncrementalAnalyzer.cs

@@ -6,10 +6,13 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Diagnostics.Telemetry;
 using Microsoft.CodeAnalysis.ErrorReporting;
 using Microsoft.CodeAnalysis.Internal.Log;
+using Microsoft.CodeAnalysis.Remote;
 using Microsoft.CodeAnalysis.Shared.Extensions;
 using Microsoft.CodeAnalysis.Shared.Options;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
 using Microsoft.CodeAnalysis.SolutionCrawler;
 using Microsoft.CodeAnalysis.Workspaces.Diagnostics;
 using Roslyn.Utilities;
@@ -59,6 +62,9 @@ private async Task AnalyzeDocumentForKindAsync(Document document, AnalysisKind k
 
                     RaiseDocumentDiagnosticsIfNeeded(document, stateSet, kind, result.OldItems, result.Items);
                 }
+
+                var asyncToken = Owner.Listener.BeginAsyncOperation(nameof(AnalyzeDocumentForKindAsync));
+                var _ = ReportAnalyzerPerformanceAsync(document, analyzerDriverOpt, cancellationToken).CompletesAsyncOperation(asyncToken);
             }
             catch (Exception e) when (FatalError.ReportUnlessCanceled(e))
             {
@@ -429,5 +435,57 @@ private void RaiseProjectDiagnosticsRemoved(StateSet stateSet, ProjectId project
 
             RaiseDiagnosticsRemoved(projectId, nullSolution, stateSet, raiseEvents);
         }
+
+        private async Task ReportAnalyzerPerformanceAsync(Document document, CompilationWithAnalyzers analyzerDriverOpt, CancellationToken cancellationToken)
+        {
+            try
+            {
+                if (analyzerDriverOpt == null)
+                {
+                    return;
+                }
+
+                var client = await document.Project.Solution.Workspace.TryGetRemoteHostClientAsync(cancellationToken).ConfigureAwait(false);
+                if (client == null)
+                {
+                    // no remote support
+                    return;
+                }
+
+                cancellationToken.ThrowIfCancellationRequested();
+
+                using (var pooledObject = SharedPools.Default<Dictionary<DiagnosticAnalyzer, AnalyzerTelemetryInfo>>().GetPooledObject())
+                {
+                    var containsData = false;
+                    foreach (var analyzer in analyzerDriverOpt.Analyzers)
+                    {
+                        var telemetryInfo = await analyzerDriverOpt.GetAnalyzerTelemetryInfoAsync(analyzer, cancellationToken).ConfigureAwait(false);
+                        if (!containsData && telemetryInfo.ExecutionTime.Ticks > 0)
+                        {
+                            // this is unfortunate tweak due to how GetAnalyzerTelemetryInfoAsync works when analyzers are asked
+                            // one by one rather than in bulk.
+                            containsData = true;
+                        }
+
+                        pooledObject.Object.Add(analyzer, telemetryInfo);
+                    }
+
+                    if (!containsData)
+                    {
+                        // looks like there is no new data from driver. skip reporting.
+                        return;
+                    }
+
+                    await client.TryRunCodeAnalysisRemoteAsync(
+                        nameof(IRemoteDiagnosticAnalyzerService.ReportAnalyzerPerformance),
+                        new object[] { pooledObject.Object.ToAnalyzerPerformanceInfo(Owner), /* unit count */ 1 },
+                        cancellationToken).ConfigureAwait(false);
+                }
+            }
+            catch (Exception ex) when (FatalError.ReportWithoutCrashUnlessCanceled(ex))
+            {
+                // this is fire and forget method
+            }
+        }
     }
 }

src/Features/Core/Portable/Diagnostics/EngineV2/ICodeAnalysisDiagnosticAnalyzerExecutor.cs

-// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
-
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.CodeAnalysis.Host;
-using Microsoft.CodeAnalysis.Workspaces.Diagnostics;
-
-namespace Microsoft.CodeAnalysis.Diagnostics.EngineV2
-{
-    /// <summary>
-    /// Interface to run DiagnosticAnalyzers. Implementation of this interface should be 
-    /// able to run analyzers that can run in command line (Host agnostic DiagnosticAnalyzers)
-    /// 
-    /// How and where analyzers run depends on the implementation of this interface
-    /// </summary>
-    internal interface ICodeAnalysisDiagnosticAnalyzerExecutor : IWorkspaceService
-    {
-        Task<DiagnosticAnalysisResultMap<DiagnosticAnalyzer, DiagnosticAnalysisResult>> AnalyzeAsync(CompilationWithAnalyzers analyzerDriver, Project project, bool forcedAnalysis, CancellationToken cancellationToken);
-    }
-}

src/Features/Core/Portable/Diagnostics/IRemoteDiagnosticAnalyzerService.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.CodeAnalysis.Diagnostics
+{
+    interface IRemoteDiagnosticAnalyzerService
+    {
+        Task CalculateDiagnosticsAsync(DiagnosticArguments arguments, string streamName, CancellationToken cancellationToken);
+        void ReportAnalyzerPerformance(List<AnalyzerPerformanceInfo> snapshot, int unitCount, CancellationToken cancellationToken);
+    }
+
+    internal struct AnalyzerPerformanceInfo
+    {
+        public readonly string AnalyzerId;
+        public readonly bool BuiltIn;
+        public readonly TimeSpan TimeSpan;
+
+        public AnalyzerPerformanceInfo(string analyzerid, bool builtIn, TimeSpan timeSpan)
+        {
+            AnalyzerId = analyzerid;
+            BuiltIn = builtIn;
+            TimeSpan = timeSpan;
+        }
+    }
+}

src/Features/Core/Portable/Diagnostics/Log/DiagnosticAnalyzerLogger.cs

@@ -52,7 +52,7 @@ public static void LogAnalyzerCrashCount(DiagnosticAnalyzer analyzer, Exception
             }
 
             // TODO: once we create description manager, pass that into here.
-            bool telemetry = DiagnosticAnalyzerLogger.AllowsTelemetry(null, analyzer, projectId);
+            bool telemetry = DiagnosticAnalyzerLogger.AllowsTelemetry(analyzer, null);
             var tuple = ValueTuple.Create(telemetry, analyzer.GetType(), ex.GetType());
             logAggregator.IncreaseCount(tuple);
         }
@@ -137,28 +137,34 @@ public static void LogAnalyzerTypeCountSummary(int correlationId, DiagnosticLogA
             }
         }
 
-        public static bool AllowsTelemetry(DiagnosticAnalyzerService service, DiagnosticAnalyzer analyzer, ProjectId projectIdOpt)
+        public static bool AllowsTelemetry(DiagnosticAnalyzer analyzer, IDiagnosticAnalyzerService serviceOpt = null)
         {
             if (s_telemetryCache.TryGetValue(analyzer, out var value))
             {
                 return value.Value;
             }
 
-            return s_telemetryCache.GetValue(analyzer, a => new StrongBox<bool>(CheckTelemetry(service, a))).Value;
+            return s_telemetryCache.GetValue(analyzer, a => new StrongBox<bool>(CheckTelemetry(a, serviceOpt))).Value;
         }
 
-        private static bool CheckTelemetry(DiagnosticAnalyzerService service, DiagnosticAnalyzer analyzer)
+        private static bool CheckTelemetry(DiagnosticAnalyzer analyzer, IDiagnosticAnalyzerService serviceOpt)
         {
             if (analyzer.IsCompilerAnalyzer())
             {
                 return true;
             }
 
+            if (analyzer is IBuiltInAnalyzer)
+            {
+                // if it is builtin analyzer, telemetry is always allowed
+                return true;
+            }
+
             ImmutableArray<DiagnosticDescriptor> diagDescriptors;
             try
             {
                 // SupportedDiagnostics is potentially user code and can throw an exception.
-                diagDescriptors = service != null ? service.GetDiagnosticDescriptors(analyzer) : analyzer.SupportedDiagnostics;
+                diagDescriptors = serviceOpt != null ? serviceOpt.GetDiagnosticDescriptors(analyzer) : analyzer.SupportedDiagnostics;
             }
             catch (Exception)
             {

src/Features/Core/Portable/Diagnostics/Log/DiagnosticLogAggregator.cs

@@ -41,7 +41,7 @@ public DiagnosticLogAggregator(DiagnosticAnalyzerService owner)
 
         public void UpdateAnalyzerTypeCount(DiagnosticAnalyzer analyzer, AnalyzerTelemetryInfo analyzerTelemetryInfo, Project projectOpt)
         {
-            var telemetry = DiagnosticAnalyzerLogger.AllowsTelemetry(_owner, analyzer, projectOpt?.Id);
+            var telemetry = DiagnosticAnalyzerLogger.AllowsTelemetry(analyzer, _owner);
 
             ImmutableInterlocked.AddOrUpdate(
                 ref _analyzerInfoMap,

src/VisualStudio/Core/Test.Next/Remote/JsonConverterTests.cs

@@ -139,6 +139,14 @@ public void TestPinnedSolutionInfo()
             });
         }
 
+        [Fact, Trait(Traits.Feature, Traits.Features.RemoteHost)]
+        public void TestAnalyzerPerformanceInfo()
+        {
+            VerifyJsonSerialization(
+                new AnalyzerPerformanceInfo("testAnalyzer", builtIn: false, timeSpan: TimeSpan.FromMilliseconds(12345)),
+                (x, y) => (x.AnalyzerId == y.AnalyzerId && x.BuiltIn == y.BuiltIn && x.TimeSpan == y.TimeSpan) ? 0 : 1);
+        }
+
         private static void VerifyJsonSerialization<T>(T value, Comparison<T> equality = null)
         {
             var serializer = new JsonSerializer();

src/VisualStudio/Core/Test.Next/Services/PerformanceTrackerServiceTests.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.Remote.Diagnostics;
+using Xunit;
+
+namespace Roslyn.VisualStudio.Next.UnitTests.Services
+{
+    public class PerformanceTrackerServiceTests
+    {
+        [Fact]
+        public void TestTooFewSamples()
+        {
+            // minimum sample is 100
+            var badAnalyzers = GetBadAnalyzers(@"TestFiles\analyzer_input.csv", to: 80);
+
+            Assert.Empty(badAnalyzers);
+        }
+
+        [Fact]
+        public void TestTracking()
+        {
+            var badAnalyzers = GetBadAnalyzers(@"TestFiles\analyzer_input.csv", to: 200);
+
+            VerifyBadAnalyzer(badAnalyzers[0], "CSharpRemoveUnnecessaryCastDiagnosticAnalyzer", 101.244432561581, 54.48, 21.8163001442628);
+            VerifyBadAnalyzer(badAnalyzers[1], "CSharpInlineDeclarationDiagnosticAnalyzer", 49.9389715502954, 26.6686092715232, 9.2987133054884);
+            VerifyBadAnalyzer(badAnalyzers[2], "VisualBasicRemoveUnnecessaryCastDiagnosticAnalyzer", 42.0967360557792, 23.277619047619, 7.25464266261805);
+        }
+
+        [Fact]
+        public void TestTrackingMaxSample()
+        {
+            var badAnalyzers = GetBadAnalyzers(@"TestFiles\analyzer_input.csv", to: 300);
+
+            VerifyBadAnalyzer(badAnalyzers[0], "CSharpRemoveUnnecessaryCastDiagnosticAnalyzer", 85.6039521236341, 58.4542358078603, 18.4245217226717);
+            VerifyBadAnalyzer(badAnalyzers[1], "VisualBasic.UseAutoProperty.UseAutoPropertyAnalyzer", 45.0918385052674, 29.0622535211268, 9.13728667060397);
+            VerifyBadAnalyzer(badAnalyzers[2], "CSharpInlineDeclarationDiagnosticAnalyzer", 42.2014208750466, 28.7935371179039, 7.99261581900397);
+        }
+
+        [Fact]
+        public void TestTrackingRolling()
+        {
+            // data starting to rolling at 300 data points
+            var badAnalyzers = GetBadAnalyzers(@"TestFiles\analyzer_input.csv", to: 400);
+
+            VerifyBadAnalyzer(badAnalyzers[0], "CSharpRemoveUnnecessaryCastDiagnosticAnalyzer", 76.2748443491852, 51.1698695652174, 17.3819563479479);
+            VerifyBadAnalyzer(badAnalyzers[1], "VisualBasic.UseAutoProperty.UseAutoPropertyAnalyzer", 43.5700167914005, 29.2597857142857, 9.21213873850298);
+            VerifyBadAnalyzer(badAnalyzers[2], "InlineDeclaration.CSharpInlineDeclarationDiagnosticAnalyzer", 36.4336594793033, 23.9764782608696, 7.43956680199015);
+        }
+
+        [Fact]
+        public void TestBadAnalyzerInfoPII()
+        {
+            var badAnalyzer1 = new ExpensiveAnalyzerInfo(true, "test", 0.1, 0.1, 0.1);
+            Assert.True(badAnalyzer1.PIISafeAnalyzerId == badAnalyzer1.AnalyzerId);
+            Assert.True(badAnalyzer1.PIISafeAnalyzerId == "test");
+
+            var badAnalyzer2 = new ExpensiveAnalyzerInfo(false, "test", 0.1, 0.1, 0.1);
+            Assert.True(badAnalyzer2.PIISafeAnalyzerId == badAnalyzer2.AnalyzerIdHash);
+            Assert.True(badAnalyzer2.PIISafeAnalyzerId == "test".GetHashCode().ToString());
+        }
+
+        private void VerifyBadAnalyzer(ExpensiveAnalyzerInfo analyzer, string analyzerId, double lof, double mean, double stddev)
+        {
+            Assert.True(analyzer.PIISafeAnalyzerId.IndexOf(analyzerId, StringComparison.OrdinalIgnoreCase) >= 0);
+            Assert.Equal(analyzer.LocalOutlierFactor, lof, precision: 4);
+            Assert.Equal(analyzer.Average, mean, precision: 4);
+            Assert.Equal(analyzer.AdjustedStandardDeviation, stddev, precision: 4);
+        }
+
+        private List<ExpensiveAnalyzerInfo> GetBadAnalyzers(string testFileName, int to)
+        {
+            var testFile = ReadTestFile(testFileName);
+
+            var (matrix, dataCount) = CreateMatrix(testFile);
+
+            to = Math.Min(to, dataCount);
+
+            var service = new PerformanceTrackerService(minLOFValue: 0, averageThreshold: 0, stddevThreshold: 0);
+
+            for (var i = 0; i < to; i++)
+            {
+                service.AddSnapshot(CreateSnapshots(matrix, i), unitCount: 100);
+            }
+
+            var badAnalyzerInfo = new List<ExpensiveAnalyzerInfo>();
+            service.GenerateReport(badAnalyzerInfo);
+
+            return badAnalyzerInfo;
+        }
+
+        private IEnumerable<AnalyzerPerformanceInfo> CreateSnapshots(Dictionary<string, double[]> matrix, int index)
+        {
+            foreach (var kv in matrix)
+            {
+                var timeSpan = kv.Value[index];
+                if (double.IsNaN(timeSpan))
+                {
+                    continue;
+                }
+
+                yield return new AnalyzerPerformanceInfo(kv.Key, true, TimeSpan.FromMilliseconds(timeSpan));
+            }
+        }
+
+        private (Dictionary<string, double[]> matrix, int dataCount) CreateMatrix(string testFile)
+        {
+            var matrix = new Dictionary<string, double[]>();
+
+            var lines = testFile.Split('\n');
+            var expectedDataCount = GetExpectedDataCount(lines[0]);
+
+            for (var i = 1; i < lines.Length; i++)
+            {
+                if (lines[i].Trim().Length == 0)
+                {
+                    continue;
+                }
+
+                var data = SkipAnalyzerId(lines[i]).Split(',');
+                Assert.Equal(data.Length, expectedDataCount);
+
+                var analyzerId = GetAnalyzerId(lines[i]);
+
+                var timeSpans = new double[expectedDataCount];
+                for (var j = 0; j < data.Length; j++)
+                {
+                    double result;
+                    if (!double.TryParse(data[j], out result))
+                    {
+                        // no data for this analyzer for this particular run
+                        result = double.NaN;
+                    }
+
+                    timeSpans[j] = result;
+                }
+
+                matrix[analyzerId] = timeSpans;
+            }
+
+            return (matrix, expectedDataCount);
+        }
+
+        private string GetAnalyzerId(string line)
+        {
+            return line.Substring(1, line.LastIndexOf('"') - 1);
+        }
+
+        private int GetExpectedDataCount(string header)
+        {
+            var data = header.Split(',');
+            return data.Length - 1;
+        }
+
+        private string SkipAnalyzerId(string line)
+        {
+            return line.Substring(line.LastIndexOf('"') + 2);
+        }
+
+        private string ReadTestFile(string name)
+        {
+            var assembly = typeof(PerformanceTrackerServiceTests).Assembly;
+            var resourceName = GetResourceName(assembly, name);
+
+            using (var stream = assembly.GetManifestResourceStream(resourceName))
+            {
+                if (stream == null)
+                {
+                    throw new InvalidOperationException($"Resource '{resourceName}' not found in {assembly.FullName}.");
+                }
+
+                using (var reader = new StreamReader(stream))
+                {
+                    return reader.ReadToEnd();
+                }
+            }
+        }
+
+        private static string GetResourceName(Assembly assembly, string name)
+        {
+            var convert = name.Replace(@"\", ".");
+
+            return assembly.GetManifestResourceNames().Where(n => n.EndsWith(convert, StringComparison.OrdinalIgnoreCase)).First();
+        }
+    }
+}

src/VisualStudio/Core/Test.Next/Services/VisualStudioDiagnosticAnalyzerExecutorTests.cs

@@ -156,7 +156,7 @@ void Method()
                 // run analysis
                 var project = workspace.CurrentSolution.Projects.First();
 
-                var executor = (ICodeAnalysisDiagnosticAnalyzerExecutor)new DiagnosticAnalyzerExecutor(new MyUpdateSource(workspace)).CreateService(workspace.Services);
+                var executor = new DiagnosticIncrementalAnalyzer.InProcOrRemoteHostAnalyzerRunner(owner: null, hostDiagnosticUpdateSource: new MyUpdateSource(workspace));
                 var analyzerDriver = (await project.GetCompilationAsync()).WithAnalyzers(
                         analyzerReference.GetAnalyzers(project.Language).Where(a => a.GetType() == analyzerType).ToImmutableArray(),
                         new WorkspaceAnalyzerOptions(project.AnalyzerOptions, project.Solution.Options, project.Solution));
@@ -202,7 +202,7 @@ void Method()
                 // run analysis
                 var project = workspace.CurrentSolution.Projects.First().AddAnalyzerReference(analyzerReference);
 
-                var executor = (ICodeAnalysisDiagnosticAnalyzerExecutor)new DiagnosticAnalyzerExecutor(new MyUpdateSource(workspace)).CreateService(workspace.Services);
+                var executor = new DiagnosticIncrementalAnalyzer.InProcOrRemoteHostAnalyzerRunner(owner: null, hostDiagnosticUpdateSource: new MyUpdateSource(workspace));
                 var analyzerDriver = (await project.GetCompilationAsync()).WithAnalyzers(
                         analyzerReference.GetAnalyzers(project.Language).Where(a => a.GetType() == analyzerType).ToImmutableArray(),
                         new WorkspaceAnalyzerOptions(project.AnalyzerOptions, project.Solution.Options, project.Solution));
@@ -219,7 +219,7 @@ void Method()
 
         private static async Task<DiagnosticAnalysisResult> AnalyzeAsync(TestWorkspace workspace, ProjectId projectId, Type analyzerType, CancellationToken cancellationToken = default(CancellationToken))
         {
-            var executor = (ICodeAnalysisDiagnosticAnalyzerExecutor)new DiagnosticAnalyzerExecutor(new MyUpdateSource(workspace)).CreateService(workspace.Services);
+            var executor = new DiagnosticIncrementalAnalyzer.InProcOrRemoteHostAnalyzerRunner(owner: null, hostDiagnosticUpdateSource: new MyUpdateSource(workspace));
 
             var analyzerReference = new AnalyzerFileReference(analyzerType.Assembly.Location, new TestAnalyzerAssemblyLoader());
             var project = workspace.CurrentSolution.GetProject(projectId).AddAnalyzerReference(analyzerReference);

src/VisualStudio/Core/Test.Next/VisualStudioTest.Next.csproj

@@ -94,4 +94,7 @@
   <ItemGroup>
     <InternalsVisibleToMoq Include="DynamicProxyGenAssembly2" />
   </ItemGroup>
-</Project>
+  <ItemGroup>
+    <EmbeddedResource Include="TestFiles\analyzer_input.csv" />
+  </ItemGroup>
+</Project>
\ No newline at end of file

src/Workspaces/Core/Portable/Log/FunctionId.cs

@@ -403,7 +403,10 @@ internal enum FunctionId
         MetadataOnlyImage_EmitFailure,
         LiveTableDataSource_OnDiagnosticsUpdated,
         Experiment_KeybindingsReset,
-        
+        Diagnostics_GeneratePerformaceReport,
+        Diagnostics_BadAnalyzer,
+        CodeAnalysisService_ReportAnalyzerPerformance,
+        PerformanceTrackerService_AddSnapshot,
         AbstractProject_SetIntelliSenseBuild,
         AbstractProject_Created,
         AbstractProject_PushedToWorkspace,

src/Workspaces/Core/Portable/Remote/RemoteHostClientExtensions.cs

@@ -225,6 +225,14 @@ public static bool IsOutOfProcessEnabled(this Workspace workspace, Option<bool>
             this RemoteHostClient client, Solution solution, string targetName, object[] arguments, CancellationToken cancellationToken)
             => TryRunRemoteAsync<T>(client, WellKnownServiceHubServices.CodeAnalysisService, solution, targetName, arguments, cancellationToken);
 
+        public static Task<bool> TryRunCodeAnalysisRemoteAsync(
+            this RemoteHostClient client, string targetName, object argument, CancellationToken cancellationToken)
+            => TryRunRemoteAsync(client, WellKnownServiceHubServices.CodeAnalysisService, targetName, new object[] { argument }, cancellationToken);
+
+        public static Task<bool> TryRunCodeAnalysisRemoteAsync(
+            this RemoteHostClient client, string targetName, object[] arguments, CancellationToken cancellationToken)
+            => TryRunRemoteAsync(client, WellKnownServiceHubServices.CodeAnalysisService, targetName, arguments, cancellationToken);
+
         /// <summary>
         /// Synchronize given solution as primary workspace solution in remote host
         /// </summary>

src/Workspaces/Core/Portable/Remote/WellKnownServiceHubServices.cs

@@ -25,7 +25,5 @@ public static void Set64bit(bool x64)
 
         public const string ServiceHubServiceBase_Initialize = "Initialize";
         public const string AssetService_RequestAssetAsync = "RequestAssetAsync";
-
-        public const string CodeAnalysisService_CalculateDiagnosticsAsync = "CalculateDiagnosticsAsync";
     }
 }

src/Workspaces/Remote/Core/Diagnostics/DiagnosticComputer.cs

@@ -10,6 +10,7 @@
 using Microsoft.CodeAnalysis.Diagnostics.Telemetry;
 using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Options;
+using Microsoft.CodeAnalysis.Shared.Extensions;
 using Microsoft.CodeAnalysis.Workspaces.Diagnostics;
 using Roslyn.Utilities;
 
@@ -19,11 +20,15 @@ internal class DiagnosticComputer
     {
         private readonly Project _project;
         private readonly Dictionary<DiagnosticAnalyzer, HashSet<DiagnosticData>> _exceptions;
+        private readonly IPerformanceTrackerService _performanceTracker;
 
         public DiagnosticComputer(Project project)
         {
             _project = project;
             _exceptions = new Dictionary<DiagnosticAnalyzer, HashSet<DiagnosticData>>();
+
+            // we only track performance from primary branch. all forked branch we don't care such as preview.
+            _performanceTracker = project.IsFromPrimaryBranch() ? project.Solution.Workspace.Services.GetService<IPerformanceTrackerService>() : null;
         }
 
         public async Task<DiagnosticAnalysisResultMap<string, DiagnosticAnalysisResultBuilder>> GetDiagnosticsAsync(
@@ -87,6 +92,13 @@ public DiagnosticComputer(Project project)
                 // PERF: Run all analyzers at once using the new GetAnalysisResultAsync API.
                 var analysisResult = await analyzerDriver.GetAnalysisResultAsync(cancellationToken).ConfigureAwait(false);
 
+                // record performance if tracker is available
+                if (_performanceTracker != null)
+                {
+                    // +1 to include project itself
+                    _performanceTracker.AddSnapshot(analysisResult.AnalyzerTelemetryInfo.ToAnalyzerPerformanceInfo(), _project.DocumentIds.Count + 1);
+                }
+
                 var builderMap = analysisResult.ToResultBuilderMap(_project, VersionStamp.Default, compilation, analysisResult.Analyzers, cancellationToken);
 
                 return DiagnosticAnalysisResultMap.Create(

src/Workspaces/Remote/Core/Diagnostics/IPerformanceTrackerService.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.Host;
+
+namespace Microsoft.CodeAnalysis.Remote.Diagnostics
+{
+    internal interface IPerformanceTrackerService : IWorkspaceService
+    {
+        void AddSnapshot(IEnumerable<AnalyzerPerformanceInfo> snapshot, int unitCount);
+        void GenerateReport(List<ExpensiveAnalyzerInfo> badAnalyzers);
+
+        event EventHandler SnapshotAdded;
+    }
+
+    internal struct ExpensiveAnalyzerInfo
+    {
+        public readonly bool BuiltIn;
+        public readonly string AnalyzerId;
+        public readonly string AnalyzerIdHash;
+        public readonly double LocalOutlierFactor;
+        public readonly double Average;
+        public readonly double AdjustedStandardDeviation;
+
+        public ExpensiveAnalyzerInfo(bool builtIn, string analyzerId, double lof_value, double average, double stddev) : this()
+        {
+            BuiltIn = builtIn;
+            AnalyzerId = analyzerId;
+            AnalyzerIdHash = analyzerId.GetHashCode().ToString();
+            LocalOutlierFactor = lof_value;
+            Average = average;
+            AdjustedStandardDeviation = stddev;
+        }
+
+        public string PIISafeAnalyzerId => BuiltIn ? AnalyzerId : AnalyzerIdHash;
+    }
+}

src/Workspaces/Remote/Core/Diagnostics/PerformanceQueue.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Roslyn.Utilities;
+
+namespace Microsoft.CodeAnalysis.Remote.Diagnostics
+{
+    /// <summary>
+    /// This queue hold onto raw performance data. this type itself is not thread safe. the one who uses this type
+    /// should take care of that.
+    /// </summary>
+    /// <threadsafety static="false" instance="false"/>
+    internal class PerformanceQueue
+    {
+        // we need at least 100 samples for result to be stable
+        private const int MinSampleSize = 100;
+
+        private readonly int _maxSampleSize;
+        private readonly LinkedList<Snapshot> _snapshots;
+
+        public PerformanceQueue(int maxSampleSize)
+        {
+            Contract.ThrowIfFalse(maxSampleSize > MinSampleSize);
+
+            _maxSampleSize = maxSampleSize;
+            _snapshots = new LinkedList<Snapshot>();
+        }
+
+        public int Count => _snapshots.Count;
+
+        public void Add(IEnumerable<(string analyzerId, TimeSpan timeSpan)> rawData, int unitCount)
+        {
+            if (_snapshots.Count < _maxSampleSize)
+            {
+                _snapshots.AddLast(new Snapshot(rawData, unitCount));
+            }
+            else
+            {
+                // remove the first one
+                var first = _snapshots.First;
+                _snapshots.RemoveFirst();
+
+                // update data to new data and put it back
+                first.Value.Update(rawData, unitCount);
+                _snapshots.AddLast(first);
+            }
+        }
+
+        public void GetPerformanceData(Dictionary<string, (double average, double stddev)> aggregatedPerformanceDataPerAnalyzer)
+        {
+            if (_snapshots.Count < MinSampleSize)
+            {
+                // we don't have enough data to report this
+                return;
+            }
+
+            using (var pooledMap = SharedPools.Default<Dictionary<int, string>>().GetPooledObject())
+            using (var pooledSet = SharedPools.Default<HashSet<int>>().GetPooledObject())
+            using (var pooledList = SharedPools.Default<List<double>>().GetPooledObject())
+            {
+                var reverseMap = pooledMap.Object;
+                AnalyzerNumberAssigner.Instance.GetReverseMap(reverseMap);
+
+                var analyzerSet = pooledSet.Object;
+
+                // get all analyzers
+                foreach (var snapshot in _snapshots)
+                {
+                    snapshot.AppendAnalyzers(analyzerSet);
+                }
+
+                var list = pooledList.Object;
+
+                // calculate aggregated data per analyzer
+                foreach (var assignedAnalyzerNumber in analyzerSet)
+                {
+                    foreach (var snapshot in _snapshots)
+                    {
+                        var timeSpan = snapshot.GetTimeSpanInMillisecond(assignedAnalyzerNumber);
+                        if (timeSpan == null)
+                        {
+                            // not all snapshot contains all analyzers
+                            continue;
+                        }
+
+                        list.Add(timeSpan.Value);
+                    }
+
+                    // data is only stable once we have more than certain set
+                    // of samples
+                    if (list.Count < MinSampleSize)
+                    {
+                        continue;
+                    }
+
+                    // set performance data
+                    aggregatedPerformanceDataPerAnalyzer[reverseMap[assignedAnalyzerNumber]] = GetAverageAndAdjustedStandardDeviation(list);
+
+                    list.Clear();
+                }
+            }
+        }
+
+        private (double average, double stddev) GetAverageAndAdjustedStandardDeviation(List<double> data)
+        {
+            var average = data.Average();
+            var stddev = Math.Sqrt(data.Select(ms => Math.Pow(ms - average, 2)).Average());
+            var squareLength = Math.Sqrt(data.Count);
+
+            return (average, stddev / squareLength);
+        }
+
+        private class Snapshot
+        {
+            /// <summary>
+            /// Raw performance data. 
+            /// Keyed by analyzer unique number got from AnalyzerNumberAssigner.
+            /// Value is delta (TimeSpan - minSpan) among span in this snapshot
+            /// </summary>
+            private readonly Dictionary<int, double> _performanceMap;
+
+            public Snapshot(IEnumerable<(string analyzerId, TimeSpan timeSpan)> snapshot, int unitCount) :
+                this(Convert(snapshot), unitCount)
+            {
+            }
+
+            public Snapshot(IEnumerable<(int assignedAnalyzerNumber, TimeSpan timeSpan)> rawData, int unitCount)
+            {
+                _performanceMap = new Dictionary<int, double>();
+
+                Reset(_performanceMap, rawData, unitCount);
+            }
+
+            public void Update(IEnumerable<(string analyzerId, TimeSpan timeSpan)> rawData, int unitCount)
+            {
+                Reset(_performanceMap, Convert(rawData), unitCount);
+            }
+
+            public void AppendAnalyzers(HashSet<int> analyzerSet)
+            {
+                analyzerSet.UnionWith(_performanceMap.Keys);
+            }
+
+            public double? GetTimeSpanInMillisecond(int assignedAnalyzerNumber)
+            {
+                if (!_performanceMap.TryGetValue(assignedAnalyzerNumber, out var value))
+                {
+                    return null;
+                }
+
+                return value;
+            }
+
+            private void Reset(
+                Dictionary<int, double> map, IEnumerable<(int assignedAnalyzerNumber, TimeSpan timeSpan)> rawData, int fileCount)
+            {
+                // get smallest timespan in the snapshot
+                var minSpan = rawData.Select(kv => kv.timeSpan).Min();
+
+                // for now, we just clear the map, if reusing dictionary blindly became an issue due to
+                // dictionary grew too big, then we need to do a bit more work to determine such case
+                // and re-create new dictionary
+                map.Clear();
+
+                // map is normalized to current timespan - min timspan of the snapshot
+                foreach (var (assignedAnalyzerNumber, timeSpan) in rawData)
+                {
+                    map[assignedAnalyzerNumber] = (timeSpan.TotalMilliseconds - minSpan.TotalMilliseconds) / fileCount;
+                }
+            }
+
+            private static IEnumerable<(int assignedAnalyzerNumber, TimeSpan timeSpan)> Convert(IEnumerable<(string analyzerId, TimeSpan timeSpan)> rawData)
+            {
+                return rawData.Select(kv => (AnalyzerNumberAssigner.Instance.GetUniqueNumber(kv.analyzerId), kv.timeSpan));
+            }
+        }
+
+        /// <summary>
+        /// Assign unique number to diagnostic analyzers
+        /// </summary>
+        private class AnalyzerNumberAssigner
+        {
+            public static readonly AnalyzerNumberAssigner Instance = new AnalyzerNumberAssigner();
+
+            private int _currentId;
+
+            // use simple approach for now. we don't expect it to grow too much. so entry added
+            // won't be removed until process goes away
+            private readonly Dictionary<string, int> _idMap;
+
+            private AnalyzerNumberAssigner()
+            {
+                _currentId = 0;
+                _idMap = new Dictionary<string, int>();
+            }
+
+            public int GetUniqueNumber(DiagnosticAnalyzer analyzer)
+            {
+                return GetUniqueNumber(analyzer.GetAnalyzerId());
+            }
+
+            public int GetUniqueNumber(string analyzerName)
+            {
+                if (!_idMap.TryGetValue(analyzerName, out var id))
+                {
+                    id = _currentId++;
+                    _idMap.Add(analyzerName, id);
+                }
+
+                return id;
+            }
+
+            public void GetReverseMap(Dictionary<int, string> reverseMap)
+            {
+                reverseMap.Clear();
+
+                foreach (var kv in _idMap)
+                {
+                    reverseMap.Add(kv.Value, kv.Key);
+                }
+            }
+        }
+    }
+}

src/Workspaces/Remote/Core/Diagnostics/PerformanceTrackerService.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Composition;
+using System.Linq;
+using System.Text;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.Host.Mef;
+using Microsoft.CodeAnalysis.Internal.Log;
+
+namespace Microsoft.CodeAnalysis.Remote.Diagnostics
+{
+    /// <summary>
+    /// Track diagnostic performance 
+    /// </summary>
+    [ExportWorkspaceService(typeof(IPerformanceTrackerService), WorkspaceKind.Host), Shared]
+    internal class PerformanceTrackerService : IPerformanceTrackerService
+    {
+        private static readonly Func<IEnumerable<AnalyzerPerformanceInfo>, int, string> s_snapshotLogger = SnapshotLogger;
+
+        private const double DefaultMinLOFValue = 20;
+        private const double DefaultAverageThreshold = 100;
+        private const double DefaultStddevThreshold = 100;
+
+        private const int SampleSize = 300;
+        private const double K_Value_Ratio = 2D / 3D;
+
+        private readonly double _minLOFValue;
+        private readonly double _averageThreshold;
+        private readonly double _stddevThreshold;
+
+        private readonly object _gate;
+        private readonly PerformanceQueue _queue;
+        private readonly ConcurrentDictionary<string, bool> _builtInMap = new ConcurrentDictionary<string, bool>(concurrencyLevel: 2, capacity: 10);
+
+        public event EventHandler SnapshotAdded;
+
+        public PerformanceTrackerService() :
+            this(DefaultMinLOFValue, DefaultAverageThreshold, DefaultStddevThreshold)
+        {
+        }
+
+        // internal for testing
+        internal PerformanceTrackerService(double minLOFValue, double averageThreshold, double stddevThreshold)
+        {
+            _minLOFValue = minLOFValue;
+            _averageThreshold = averageThreshold;
+            _stddevThreshold = stddevThreshold;
+
+            _gate = new object();
+            _queue = new PerformanceQueue(SampleSize);
+        }
+
+        public void AddSnapshot(IEnumerable<AnalyzerPerformanceInfo> snapshot, int unitCount)
+        {
+            Logger.Log(FunctionId.PerformanceTrackerService_AddSnapshot, s_snapshotLogger, snapshot, unitCount);
+
+            RecordBuiltInAnalyzers(snapshot);
+
+            lock (_gate)
+            {
+                _queue.Add(snapshot.Select(entry => (entry.AnalyzerId, entry.TimeSpan)), unitCount);
+            }
+
+            OnSnapshotAdded();
+        }
+
+        public void GenerateReport(List<ExpensiveAnalyzerInfo> badAnalyzers)
+        {
+            using (var pooledRaw = SharedPools.Default<Dictionary<string, (double average, double stddev)>>().GetPooledObject())
+            {
+                var rawPerformanceData = pooledRaw.Object;
+
+                lock (_gate)
+                {
+                    // first get raw aggregated peformance data from the queue
+                    _queue.GetPerformanceData(rawPerformanceData);
+                }
+
+                // make sure there are some data
+                if (rawPerformanceData.Count == 0)
+                {
+                    return;
+                }
+
+                using (var generator = new ReportGenerator(this, _minLOFValue, _averageThreshold, _stddevThreshold, badAnalyzers))
+                {
+                    generator.Report(rawPerformanceData);
+                }
+            }
+        }
+
+        private void RecordBuiltInAnalyzers(IEnumerable<AnalyzerPerformanceInfo> snapshot)
+        {
+            foreach (var entry in snapshot)
+            {
+                _builtInMap[entry.AnalyzerId] = entry.BuiltIn;
+            }
+        }
+
+        private bool AllowTelemetry(string analyzerId)
+        {
+            if (_builtInMap.TryGetValue(analyzerId, out var builtIn))
+            {
+                return builtIn;
+            }
+
+            return false;
+        }
+
+        private void OnSnapshotAdded()
+        {
+            SnapshotAdded?.Invoke(this, EventArgs.Empty);
+        }
+
+        private static string SnapshotLogger(IEnumerable<AnalyzerPerformanceInfo> snapshots, int unitCount)
+        {
+            using (var pooledObject = SharedPools.Default<StringBuilder>().GetPooledObject())
+            {
+                var sb = pooledObject.Object;
+
+                sb.Append(unitCount);
+
+                foreach (var snapshot in snapshots)
+                {
+                    sb.Append("|");
+                    sb.Append(snapshot.AnalyzerId);
+                    sb.Append(":");
+                    sb.Append(snapshot.BuiltIn);
+                    sb.Append(":");
+                    sb.Append(snapshot.TimeSpan.TotalMilliseconds);
+                }
+
+                sb.Append("*");
+
+                return sb.ToString();
+            }
+        }
+
+        private sealed class ReportGenerator : IDisposable, IComparer<ExpensiveAnalyzerInfo>
+        {
+            private readonly double _minLOFValue;
+            private readonly double _averageThreshold;
+            private readonly double _stddevThreshold;
+
+            private readonly PerformanceTrackerService _owner;
+            private readonly List<ExpensiveAnalyzerInfo> _badAnalyzers;
+            private readonly PooledObject<List<IDisposable>> _pooledObjects;
+
+            public ReportGenerator(
+                PerformanceTrackerService owner,
+                double minLOFValue,
+                double averageThreshold,
+                double stddevThreshold,
+                List<ExpensiveAnalyzerInfo> badAnalyzers)
+            {
+                _pooledObjects = SharedPools.Default<List<IDisposable>>().GetPooledObject();
+
+                _owner = owner;
+
+                _minLOFValue = minLOFValue;
+                _averageThreshold = averageThreshold;
+                _stddevThreshold = stddevThreshold;
+
+                _badAnalyzers = badAnalyzers;
+            }
+
+            public void Report(Dictionary<string, (double average, double stddev)> rawPerformanceData)
+            {
+                // this is implementation of local outlier factor (LOF)
+                // see the wiki (https://en.wikipedia.org/wiki/Local_outlier_factor) for more information 
+
+                // convert string (analyzerId) to index
+                var analyzerIdIndex = GetAnalyzerIdIndex(rawPerformanceData.Keys);
+
+                // now calculate normalized value per analyzer
+                var normalizedMap = GetNormalizedPerformanceMap(analyzerIdIndex, rawPerformanceData);
+
+                // get k value
+                var k_value = (int)(rawPerformanceData.Count * K_Value_Ratio);
+
+                // calculate distances
+
+                // calculate all distance first
+                var allDistances = GetAllDistances(normalizedMap);
+
+                // find k distance from all distances
+                var kDistances = GetKDistances(allDistances, k_value);
+
+                // find k nearest neighbors
+                var kNeighborIndices = GetKNeighborIndices(allDistances, kDistances);
+
+                var analyzerCount = kNeighborIndices.Count;
+                for (var index = 0; index < analyzerCount; index++)
+                {
+                    var analyzerId = analyzerIdIndex[index];
+
+                    // if result performance is lower than our threshold, don't need to calcuate
+                    // LOF value for the analyzer
+                    var rawData = rawPerformanceData[analyzerId];
+                    if (rawData.average <= _averageThreshold && rawData.stddev <= _stddevThreshold)
+                    {
+                        continue;
+                    }
+
+                    // possible bad analyzer, calcuate LOF
+                    var lof_value = TryGetLocalOutlierFactor(allDistances, kNeighborIndices, kDistances, index);
+                    if (!lof_value.HasValue)
+                    {
+                        // this analyzer doesn't have lof value
+                        continue;
+                    }
+
+                    if (lof_value <= _minLOFValue)
+                    {
+                        // this doesn't stand out from other analyzers
+                        continue;
+                    }
+
+                    // report found possible bad analyzers
+                    _badAnalyzers.Add(new ExpensiveAnalyzerInfo(_owner.AllowTelemetry(analyzerId), analyzerId, lof_value.Value, rawData.average, rawData.stddev));
+                }
+
+                _badAnalyzers.Sort(this);
+            }
+
+            private double? TryGetLocalOutlierFactor(
+                List<List<double>> allDistances, List<List<int>> kNeighborIndices, List<double> kDistances, int analyzerIndex)
+            {
+                var rowKNeighborsIndices = kNeighborIndices[analyzerIndex];
+                if (rowKNeighborsIndices.Count == 0)
+                {
+                    // nothing to calculate if there is no neighbor to compare
+                    return null;
+                }
+
+                var lrda = TryGetLocalReachabilityDensity(allDistances, kNeighborIndices, kDistances, analyzerIndex);
+                if (!lrda.HasValue)
+                {
+                    // can't calculate reachability for the analyzer. can't calculate lof for this analyzer
+                    return null;
+                }
+
+                var lrdb = 0D;
+                foreach (var neighborIndex in rowKNeighborsIndices)
+                {
+                    var reachability = TryGetLocalReachabilityDensity(allDistances, kNeighborIndices, kDistances, neighborIndex);
+                    if (!reachability.HasValue)
+                    {
+                        // this neighbor analyzer doesn't have its own neighbor. skip it
+                        continue;
+                    }
+
+                    lrdb += reachability.Value;
+                }
+
+                return (lrdb / rowKNeighborsIndices.Count) / lrda;
+            }
+
+            private double GetReachabilityDistance(
+                List<List<double>> allDistances, List<double> kDistances, int analyzerIndex1, int analyzerIndex2)
+            {
+                return Math.Max(allDistances[analyzerIndex1][analyzerIndex2], kDistances[analyzerIndex2]);
+            }
+
+            private double? TryGetLocalReachabilityDensity(
+                List<List<double>> allDistances, List<List<int>> kNeighborIndices, List<double> kDistances, int analyzerIndex)
+            {
+                var rowKNeighborsIndices = kNeighborIndices[analyzerIndex];
+                if (rowKNeighborsIndices.Count == 0)
+                {
+                    // no neighbor to get reachability
+                    return null;
+                }
+
+                var distanceSum = 0.0;
+                foreach (var neighborIndex in rowKNeighborsIndices)
+                {
+                    distanceSum += GetReachabilityDistance(allDistances, kDistances, analyzerIndex, neighborIndex);
+                }
+
+                return 1 / distanceSum / rowKNeighborsIndices.Count;
+            }
+
+            private List<List<int>> GetKNeighborIndices(List<List<double>> allDistances, List<double> kDistances)
+            {
+                var analyzerCount = kDistances.Count;
+                var kNeighborIndices = GetPooledListAndSetCapacity<List<int>>(analyzerCount);
+
+                for (var rowIndex = 0; rowIndex < analyzerCount; rowIndex++)
+                {
+                    var rowKNeighborIndices = GetPooledList<int>();
+
+                    var rowDistances = allDistances[rowIndex];
+                    var kDistance = kDistances[rowIndex];
+
+                    for (var colIndex = 0; colIndex < analyzerCount; colIndex++)
+                    {
+                        var value = rowDistances[colIndex];
+
+                        // get neighbors closer than k distance
+                        if (value > 0 && value <= kDistance)
+                        {
+                            rowKNeighborIndices.Add(colIndex);
+                        }
+                    }
+
+                    kNeighborIndices[rowIndex] = rowKNeighborIndices;
+                }
+
+                return kNeighborIndices;
+            }
+
+            private List<double> GetKDistances(List<List<double>> allDistances, int kValue)
+            {
+                var analyzerCount = allDistances.Count;
+                var kDistances = GetPooledListAndSetCapacity<double>(analyzerCount);
+                var sortedRowDistance = GetPooledList<double>();
+
+                for (var index = 0; index < analyzerCount; index++)
+                {
+                    sortedRowDistance.Clear();
+                    sortedRowDistance.AddRange(allDistances[index]);
+
+                    sortedRowDistance.Sort();
+
+                    kDistances[index] = sortedRowDistance[kValue];
+                }
+
+                return kDistances;
+            }
+
+            private List<List<double>> GetAllDistances(List<(double normaliedAverage, double normalizedStddev)> normalizedMap)
+            {
+                var analyzerCount = normalizedMap.Count;
+                var allDistances = GetPooledListAndSetCapacity<List<double>>(analyzerCount);
+
+                for (var rowIndex = 0; rowIndex < analyzerCount; rowIndex++)
+                {
+                    var rowDistances = GetPooledListAndSetCapacity<double>(analyzerCount);
+                    var rowAnalyzer = normalizedMap[rowIndex];
+
+                    for (var colIndex = 0; colIndex < analyzerCount; colIndex++)
+                    {
+                        var colAnalyzer = normalizedMap[colIndex];
+                        var distance = Math.Sqrt(Math.Pow(colAnalyzer.normaliedAverage - rowAnalyzer.normaliedAverage, 2) +
+                                                 Math.Pow(colAnalyzer.normalizedStddev - rowAnalyzer.normalizedStddev, 2));
+
+                        rowDistances[colIndex] = distance;
+                    }
+
+                    allDistances[rowIndex] = rowDistances;
+                }
+
+                return allDistances;
+            }
+
+            private List<(double normaliedAverage, double normalizedStddev)> GetNormalizedPerformanceMap(
+                List<string> analyzerIdIndex, Dictionary<string, (double average, double stddev)> rawPerformanceData)
+            {
+                var averageMin = rawPerformanceData.Values.Select(kv => kv.average).Min();
+                var averageMax = rawPerformanceData.Values.Select(kv => kv.average).Max();
+                var averageDelta = averageMax - averageMin;
+
+                var stddevMin = rawPerformanceData.Values.Select(kv => kv.stddev).Min();
+                var stddevMax = rawPerformanceData.Values.Select(kv => kv.stddev).Max();
+                var stddevDelta = stddevMax - stddevMin;
+
+                // make sure delta is not 0
+                averageDelta = averageDelta == 0 ? 1 : averageDelta;
+                stddevDelta = stddevDelta == 0 ? 1 : stddevDelta;
+
+                // calculate normalized average and stddev and convert analyzerId string to index
+                var analyzerCount = analyzerIdIndex.Count;
+                var normalizedMap = GetPooledListAndSetCapacity<(double normalizedAverage, double normalizedStddev)>(analyzerCount);
+
+                for (var index = 0; index < analyzerCount; index++)
+                {
+                    var value = rawPerformanceData[analyzerIdIndex[index]];
+                    var normalizedAverage = (value.average - averageMin) / averageDelta;
+                    var normalizedStddev = (value.stddev - stddevMin) / stddevDelta;
+
+                    normalizedMap[index] = (normalizedAverage, normalizedStddev);
+                }
+
+                return normalizedMap;
+            }
+
+            private List<string> GetAnalyzerIdIndex(IEnumerable<string> analyzerIds)
+            {
+                var analyzerIdIndex = GetPooledList<string>();
+                analyzerIdIndex.AddRange(analyzerIds);
+
+                return analyzerIdIndex;
+            }
+
+            public void Dispose()
+            {
+                foreach (var disposable in _pooledObjects.Object)
+                {
+                    disposable.Dispose();
+                }
+
+                _pooledObjects.Dispose();
+            }
+
+            private List<T> GetPooledList<T>()
+            {
+                var pooledObject = SharedPools.Default<List<T>>().GetPooledObject();
+                _pooledObjects.Object.Add(pooledObject);
+
+                return pooledObject.Object;
+            }
+
+            private List<T> GetPooledListAndSetCapacity<T>(int capacity)
+            {
+                var pooledObject = SharedPools.Default<List<T>>().GetPooledObject();
+                _pooledObjects.Object.Add(pooledObject);
+
+                for (var i = 0; i < capacity; i++)
+                {
+                    pooledObject.Object.Add(default(T));
+                }
+
+                return pooledObject.Object;
+            }
+
+            public int Compare(ExpensiveAnalyzerInfo x, ExpensiveAnalyzerInfo y)
+            {
+                if (x.LocalOutlierFactor == y.LocalOutlierFactor)
+                {
+                    return 0;
+                }
+
+                // want reversed order
+                if (x.LocalOutlierFactor - y.LocalOutlierFactor > 0)
+                {
+                    return -1;
+                }
+
+                return 1;
+            }
+        }
+    }
+}

src/Workspaces/Remote/ServiceHub/Services/CodeAnalysisService_Diagnostics.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Threading;
@@ -15,12 +16,12 @@
 namespace Microsoft.CodeAnalysis.Remote
 {
     // root level service for all Roslyn services
-    internal partial class CodeAnalysisService
+    internal partial class CodeAnalysisService : IRemoteDiagnosticAnalyzerService
     {
         /// <summary>
-        /// This is top level entry point for diagnostic calculation from client (VS).
-        /// 
-        /// This will be called by ServiceHub/JsonRpc framework
+        /// Calculate dignostics. this works differently than other ones such as todo comments or designer attribute scanner
+        /// since in proc and out of proc runs quite differently due to concurrency and due to possible amount of data
+        /// that needs to pass through between processes
         /// </summary>
         public Task CalculateDiagnosticsAsync(DiagnosticArguments arguments, string streamName, CancellationToken cancellationToken)
         {
@@ -53,6 +54,25 @@ public Task CalculateDiagnosticsAsync(DiagnosticArguments arguments, string stre
             }, cancellationToken);
         }
 
+        public void ReportAnalyzerPerformance(List<AnalyzerPerformanceInfo> snapshot, int unitCount, CancellationToken cancellationToken)
+        {
+            RunService(token =>
+            {
+                using (RoslynLogger.LogBlock(FunctionId.CodeAnalysisService_ReportAnalyzerPerformance, token))
+                {
+                    token.ThrowIfCancellationRequested();
+
+                    var service = SolutionService.PrimaryWorkspace.Services.GetService<IPerformanceTrackerService>();
+                    if (service == null)
+                    {
+                        return;
+                    }
+
+                    service.AddSnapshot(snapshot, unitCount);
+                }
+            }, cancellationToken);
+        }
+
         private async Task SerializeDiagnosticResultAsync(string streamName, DiagnosticAnalysisResultMap<string, DiagnosticAnalysisResultBuilder> result, CancellationToken cancellationToken)
         {
             using (RoslynLogger.LogBlock(FunctionId.CodeAnalysisService_SerializeDiagnosticResultAsync, GetResultLogInfo, result, cancellationToken))

src/Workspaces/Remote/ServiceHub/Services/RemoteHostService.PerformanceReporter.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.ErrorReporting;
+using Microsoft.CodeAnalysis.Internal.Log;
+using Microsoft.CodeAnalysis.Notification;
+using Microsoft.CodeAnalysis.Remote.Diagnostics;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
+using Microsoft.CodeAnalysis.SolutionCrawler;
+using RoslynLogger = Microsoft.CodeAnalysis.Internal.Log.Logger;
+
+namespace Microsoft.CodeAnalysis.Remote
+{
+    internal partial class RemoteHostService : ServiceHubServiceBase, IRemoteHostService
+    {
+        /// <summary>
+        /// Track when last time report has sent and send new report if there is update after given internal
+        /// </summary>
+        private class PerformanceReporter : GlobalOperationAwareIdleProcessor
+        {
+            private readonly SemaphoreSlim _event;
+            private readonly HashSet<string> _reported;
+
+            private readonly IPerformanceTrackerService _diagnosticAnalyzerPerformanceTracker;
+            private readonly TraceSource _logger;
+
+            public PerformanceReporter(TraceSource logger, IPerformanceTrackerService diagnosticAnalyzerPerformanceTracker, TimeSpan reportingInterval, CancellationToken shutdownToken) : base(
+                AsynchronousOperationListenerProvider.NullListener,
+                SolutionService.PrimaryWorkspace.Services.GetService<IGlobalOperationNotificationService>(),
+                (int)reportingInterval.TotalMilliseconds, shutdownToken)
+            {
+                _event = new SemaphoreSlim(initialCount: 0);
+                _reported = new HashSet<string>();
+
+                _logger = logger;
+                _diagnosticAnalyzerPerformanceTracker = diagnosticAnalyzerPerformanceTracker;
+                _diagnosticAnalyzerPerformanceTracker.SnapshotAdded += OnSnapshotAdded;
+                Start();
+            }
+
+            protected override void PauseOnGlobalOperation()
+            {
+                // we won't cancel report already running. we will just prevent
+                // new one from starting.
+            }
+
+            protected override async Task ExecuteAsync()
+            {
+                // wait for global operation such as build
+                await GlobalOperationTask.ConfigureAwait(false);
+
+                using (var pooledObject = SharedPools.Default<List<ExpensiveAnalyzerInfo>>().GetPooledObject())
+                using (RoslynLogger.LogBlock(FunctionId.Diagnostics_GeneratePerformaceReport, CancellationToken))
+                {
+                    _diagnosticAnalyzerPerformanceTracker.GenerateReport(pooledObject.Object);
+
+                    foreach (var badAnalyzerInfo in pooledObject.Object)
+                    {
+                        var newAnalyzer = _reported.Add(badAnalyzerInfo.AnalyzerId);
+                        var internalUser = WatsonReporter.IsUserMicrosoftInternal;
+
+                        // we only report same analyzer once unless it is internal user
+                        if (internalUser || newAnalyzer)
+                        {
+                            // this will report telemetry under VS. this will let us see how accurate our performance tracking is
+                            RoslynLogger.Log(FunctionId.Diagnostics_BadAnalyzer, KeyValueLogMessage.Create(m =>
+                            {
+                                // since it is telemetry, we hash analyzer name if it is not builtin analyzer
+                                m[nameof(badAnalyzerInfo.AnalyzerId)] = internalUser ? badAnalyzerInfo.AnalyzerId : badAnalyzerInfo.PIISafeAnalyzerId;
+                                m[nameof(badAnalyzerInfo.LocalOutlierFactor)] = badAnalyzerInfo.LocalOutlierFactor;
+                                m[nameof(badAnalyzerInfo.Average)] = badAnalyzerInfo.Average;
+                                m[nameof(badAnalyzerInfo.AdjustedStandardDeviation)] = badAnalyzerInfo.AdjustedStandardDeviation;
+                            }));
+                        }
+
+                        // for logging, we only log once. we log here so that we can ask users to provide this log to us
+                        // when we want to find out VS performance issue that could be caused by analyzer
+                        if (newAnalyzer)
+                        {
+                            _logger.TraceEvent(TraceEventType.Error, 0, $"[{badAnalyzerInfo.AnalyzerId} ({badAnalyzerInfo.AnalyzerIdHash})] LOF: {badAnalyzerInfo.LocalOutlierFactor}, Avg: {badAnalyzerInfo.Average}, Stddev: {badAnalyzerInfo.AdjustedStandardDeviation}");
+                        }
+                    }
+                }
+            }
+
+            protected override Task WaitAsync(CancellationToken cancellationToken)
+            {
+                return _event.WaitAsync(cancellationToken);
+            }
+
+            private void OnSnapshotAdded(object sender, EventArgs e)
+            {
+                // this acts like Monitor.Pulse. (wake up event if it is currently waiting
+                // if not, ignore. this can have race, but that's fine for this usage case)
+                // not using Monitor.Pulse since that doesn't support WaitAsync
+                if (_event.CurrentCount > 0)
+                {
+                    return;
+                }
+
+                _event.Release();
+            }
+        }
+    }
+}

src/Workspaces/Remote/ServiceHub/Services/RemoteHostService.cs

@@ -13,6 +13,7 @@
 using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Internal.Log;
 using Microsoft.CodeAnalysis.Notification;
+using Microsoft.CodeAnalysis.Remote.Diagnostics;
 using Microsoft.CodeAnalysis.Remote.Services;
 using Microsoft.CodeAnalysis.Remote.Storage;
 using Microsoft.CodeAnalysis.Storage;
@@ -29,13 +30,16 @@ namespace Microsoft.CodeAnalysis.Remote
     /// 
     /// basically, this is used to manage lifetime of the service hub.
     /// </summary>
-    internal class RemoteHostService : ServiceHubServiceBase, IRemoteHostService
+    internal partial class RemoteHostService : ServiceHubServiceBase, IRemoteHostService
     {
+        private readonly static TimeSpan s_reportInterval = TimeSpan.FromMinutes(2);
+
         // it is saved here more on debugging purpose.
         private static Func<FunctionId, bool> s_logChecker = _ => false;
 
         private string _host;
         private int _primaryInstance;
+        private PerformanceReporter _performanceReporter;
 
         static RemoteHostService()
         {
@@ -195,7 +199,7 @@ private void SetSessionInfo(Dictionary<string, object> m)
             m["InstanceId"] = _primaryInstance;
         }
 
-        private static void SetGlobalContext(int uiCultureLCID, int cultureLCID, string serializedSession)
+        private void SetGlobalContext(int uiCultureLCID, int cultureLCID, string serializedSession)
         {
             // set global telemetry session
             var session = GetTelemetrySession(serializedSession);
@@ -214,6 +218,13 @@ private static void SetGlobalContext(int uiCultureLCID, int cultureLCID, string
             // set both handler as NFW
             FatalError.Handler = WatsonReporter.Report;
             FatalError.NonFatalHandler = WatsonReporter.Report;
+
+            // start performance reporter
+            var diagnosticAnalyzerPerformanceTracker = SolutionService.PrimaryWorkspace.Services.GetService<IPerformanceTrackerService>();
+            if (diagnosticAnalyzerPerformanceTracker != null)
+            {
+                _performanceReporter = new PerformanceReporter(Logger, diagnosticAnalyzerPerformanceTracker, s_reportInterval, ShutdownCancellationToken);
+            }
         }
 
         private static void EnsureCulture(int uiCultureLCID, int cultureLCID)

src/Workspaces/Remote/ServiceHub/Shared/RoslynJsonConverter.RoslynOnly.cs

@@ -7,6 +7,7 @@
 using Microsoft.CodeAnalysis.AddImport;
 using Microsoft.CodeAnalysis.CodeActions;
 using Microsoft.CodeAnalysis.DesignerAttributes;
+using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.DocumentHighlighting;
 using Microsoft.CodeAnalysis.Packaging;
 using Microsoft.CodeAnalysis.SymbolSearch;
@@ -34,6 +35,8 @@ internal partial class AggregateJsonConverter : JsonConverter
 
             Add(builder, new ReferenceAssemblyWithTypeResultJsonConverter());
             Add(builder, new AddImportFixDataJsonConverter());
+
+            Add(builder, new AnalyzerPerformanceInfoConverter());
         }
 
         private class TodoCommentDescriptorJsonConverter : BaseJsonConverter<TodoCommentDescriptor>
@@ -73,7 +76,7 @@ protected override TodoComment ReadValue(JsonReader reader, JsonSerializer seria
                 Contract.ThrowIfFalse(reader.TokenType == JsonToken.StartObject);
 
                 // all integer is long
-                var descriptor = ReadProperty<TodoCommentDescriptor>(serializer, reader);
+                var descriptor = ReadProperty<TodoCommentDescriptor>(reader, serializer);
                 var message = ReadProperty<string>(reader);
                 var position = ReadProperty<long>(reader);
 
@@ -168,7 +171,7 @@ protected override HighlightSpan ReadValue(JsonReader reader, JsonSerializer ser
             {
                 Contract.ThrowIfFalse(reader.TokenType == JsonToken.StartObject);
 
-                var textSpan = ReadProperty<TextSpan>(serializer, reader);
+                var textSpan = ReadProperty<TextSpan>(reader, serializer);
                 var kind = (HighlightSpanKind)ReadProperty<long>(reader);
 
                 Contract.ThrowIfFalse(reader.Read());
@@ -201,7 +204,7 @@ protected override PackageWithTypeResult ReadValue(JsonReader reader, JsonSerial
                 var typeName = ReadProperty<string>(reader);
                 var version = ReadProperty<string>(reader);
                 var rank = (int)ReadProperty<long>(reader);
-                var containingNamespaceNames = ReadProperty<IList<string>>(serializer, reader);
+                var containingNamespaceNames = ReadProperty<IList<string>>(reader, serializer);
 
                 Contract.ThrowIfFalse(reader.Read());
                 Contract.ThrowIfFalse(reader.TokenType == JsonToken.EndObject);
@@ -273,7 +276,7 @@ protected override ReferenceAssemblyWithTypeResult ReadValue(JsonReader reader,
 
                 var assemblyName = ReadProperty<string>(reader);
                 var typeName = ReadProperty<string>(reader);
-                var containingNamespaceNames = ReadProperty<IList<string>>(serializer, reader);
+                var containingNamespaceNames = ReadProperty<IList<string>>(reader, serializer);
 
                 Contract.ThrowIfFalse(reader.Read());
                 Contract.ThrowIfFalse(reader.TokenType == JsonToken.EndObject);
@@ -334,14 +337,14 @@ protected override AddImportFixData ReadValue(JsonReader reader, JsonSerializer
                 Contract.ThrowIfFalse(reader.TokenType == JsonToken.StartObject);
 
                 var kind = (AddImportFixKind)ReadProperty<long>(reader);
-                var textChanges = ReadProperty<IList<TextChange>>(serializer, reader).ToImmutableArrayOrEmpty();
+                var textChanges = ReadProperty<IList<TextChange>>(reader, serializer).ToImmutableArrayOrEmpty();
                 var title = ReadProperty<string>(reader);
-                var tags = ReadProperty<IList<string>>(serializer, reader).ToImmutableArrayOrEmpty();
+                var tags = ReadProperty<IList<string>>(reader, serializer).ToImmutableArrayOrEmpty();
                 var priority = (CodeActionPriority)ReadProperty<long>(reader);
 
-                var projectReferenceToAdd = ReadProperty<ProjectId>(serializer, reader);
+                var projectReferenceToAdd = ReadProperty<ProjectId>(reader, serializer);
 
-                var portableExecutableReferenceProjectId = ReadProperty<ProjectId>(serializer, reader);
+                var portableExecutableReferenceProjectId = ReadProperty<ProjectId>(reader, serializer);
                 var portableExecutableReferenceFilePathToAdd = ReadProperty<string>(reader);
 
                 var assemblyReferenceAssemblyName = ReadProperty<string>(reader);
@@ -418,5 +421,38 @@ protected override void WriteValue(JsonWriter writer, AddImportFixData source, J
                 writer.WriteEndObject();
             }
         }
+
+        private class AnalyzerPerformanceInfoConverter : BaseJsonConverter<AnalyzerPerformanceInfo>
+        {
+            protected override AnalyzerPerformanceInfo ReadValue(JsonReader reader, JsonSerializer serializer)
+            {
+                Contract.ThrowIfFalse(reader.TokenType == JsonToken.StartObject);
+
+                var analyzerid = ReadProperty<string>(reader);
+                var builtIn = ReadProperty<bool>(reader);
+                var timeSpan = ReadProperty<TimeSpan>(reader, serializer);
+
+                Contract.ThrowIfFalse(reader.Read());
+                Contract.ThrowIfFalse(reader.TokenType == JsonToken.EndObject);
+
+                return new AnalyzerPerformanceInfo(analyzerid, builtIn, timeSpan);
+            }
+
+            protected override void WriteValue(JsonWriter writer, AnalyzerPerformanceInfo info, JsonSerializer serializer)
+            {
+                writer.WriteStartObject();
+
+                writer.WritePropertyName(nameof(AnalyzerPerformanceInfo.AnalyzerId));
+                writer.WriteValue(info.AnalyzerId);
+
+                writer.WritePropertyName(nameof(AnalyzerPerformanceInfo.BuiltIn));
+                writer.WriteValue(info.BuiltIn);
+
+                writer.WritePropertyName(nameof(AnalyzerPerformanceInfo.TimeSpan));
+                serializer.Serialize(writer, info.TimeSpan);
+
+                writer.WriteEndObject();
+            }
+        }
     }
 }

src/Workspaces/Remote/ServiceHub/Shared/RoslynJsonConverter.SolutionIdConverters.cs

@@ -87,7 +87,7 @@ protected override DocumentId ReadValue(JsonReader reader, JsonSerializer serial
 
                 Contract.ThrowIfFalse(reader.TokenType == JsonToken.StartObject);
 
-                var projectId = ReadProperty<ProjectId>(serializer, reader);
+                var projectId = ReadProperty<ProjectId>(reader, serializer);
 
                 var (id, debugName) = ReadIdAndName(reader);
 

src/Workspaces/Remote/ServiceHub/Shared/RoslynJsonConverter.cs

@@ -72,7 +72,7 @@ public sealed override void WriteJson(JsonWriter writer, object value, JsonSeria
             protected abstract T ReadValue(JsonReader reader, JsonSerializer serializer);
             protected abstract void WriteValue(JsonWriter writer, T value, JsonSerializer serializer);
 
-            protected static U ReadProperty<U>(JsonSerializer serializer, JsonReader reader)
+            protected static U ReadProperty<U>(JsonReader reader, JsonSerializer serializer)
             {
                 // read property
                 Contract.ThrowIfFalse(reader.Read());
@@ -189,7 +189,7 @@ protected override PinnedSolutionInfo ReadValue(JsonReader reader, JsonSerialize
                 // all integer is long
                 var scopeId = ReadProperty<long>(reader);
                 var fromPrimaryBranch = ReadProperty<bool>(reader);
-                var checksum = ReadProperty<Checksum>(serializer, reader);
+                var checksum = ReadProperty<Checksum>(reader, serializer);
 
                 Contract.ThrowIfFalse(reader.Read());
                 Contract.ThrowIfFalse(reader.TokenType == JsonToken.EndObject);

src/Workspaces/Remote/ServiceHub/Telemetry/WatsonReporter.cs

@@ -30,6 +30,11 @@ public static void SetTelemetrySession(TelemetrySession session)
         /// </summary>
         public static TelemetrySession SessionOpt => s_sessionOpt;
 
+        /// <summary>
+        /// Check whether current user is microsoft internal or not
+        /// </summary>
+        public static bool IsUserMicrosoftInternal => SessionOpt?.IsUserMicrosoftInternal ?? false;
+
         /// <summary>
         /// Report Non-Fatal Watson
         /// </summary>