added OpenFileOnly in IBuiltInAnalyzer and removed RunInProcess

now, all builtin analyzer that is not set to only run on open files will run in OOP.

any builtin analyzer that can't run in OOP should mark it as open file only.

for analyzers that return only hidden severity but return different severity on runtime should use the new API to control whether it want it to run full solution or not. if set to run on full solution, it will run in OOP.

src/EditorFeatures/CSharpTest/Diagnostics/DiagnosticAnalyzerDriver/DiagnosticAnalyzerDriverTests.cs

@@ -167,7 +167,7 @@ private void AccessSupportedDiagnostics(DiagnosticAnalyzer analyzer)
 
         private class ThrowingDoNotCatchDiagnosticAnalyzer<TLanguageKindEnum> : ThrowingDiagnosticAnalyzer<TLanguageKindEnum>, IBuiltInAnalyzer where TLanguageKindEnum : struct
         {
-            public bool RunInProcess => true;
+            public bool OpenFileOnly(Workspace workspace) => true;
 
             public DiagnosticAnalyzerCategory GetAnalyzerCategory()
             {

src/EditorFeatures/Core/Implementation/RenameTracking/RenameTrackingDiagnosticAnalyzer.cs

@@ -19,7 +19,7 @@ internal sealed class RenameTrackingDiagnosticAnalyzer : DiagnosticAnalyzer, IBu
         internal const string RenameToPropertyKey = "RenameTo";
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(DiagnosticDescriptor);
-        public bool RunInProcess => true;
+        public bool OpenFileOnly(Workspace workspace) => true;
 
         public override void Initialize(AnalysisContext context)
         {

src/EditorFeatures/Test/Diagnostics/AbstractSuppressionAllCodeTests.cs

@@ -135,7 +135,7 @@ internal class Analyzer : DiagnosticAnalyzer, IBuiltInAnalyzer
             private readonly DiagnosticDescriptor _descriptor =
                     new DiagnosticDescriptor("TestId", "Test", "Test", "Test", DiagnosticSeverity.Warning, isEnabledByDefault: true);
 
-            public bool RunInProcess => true;
+            public bool OpenFileOnly(Workspace workspace) => true;
 
             public ImmutableArray<SyntaxNode> AllNodes { get; set; }
 

src/Features/CSharp/Portable/AddBraces/CSharpAddBracesDiagnosticAnalyzer.cs

@@ -26,7 +26,7 @@ internal sealed class CSharpAddBracesDiagnosticAnalyzer : DiagnosticAnalyzer, IB
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics { get; } = ImmutableArray.Create(s_descriptor);
 
-        public bool RunInProcess => false;
+        public bool OpenFileOnly(Workspace workspace) => false;
 
         public override void Initialize(AnalysisContext context)
         {

src/Features/CSharp/Portable/Diagnostics/Analyzers/CSharpTypeStyleDiagnosticAnalyzerBase.cs

@@ -38,7 +38,7 @@ public CSharpTypeStyleDiagnosticAnalyzerBase(string diagnosticId, LocalizableStr
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(CreateDiagnosticDescriptor(DiagnosticSeverity.Hidden));
 
         public DiagnosticAnalyzerCategory GetAnalyzerCategory() => DiagnosticAnalyzerCategory.SemanticSpanAnalysis;
-        public bool RunInProcess => true;
+        public bool OpenFileOnly(Workspace workspace) => true;
 
         public override void Initialize(AnalysisContext context)
         {

src/Features/CSharp/Portable/InvokeDelegateWithConditionalAccess/InvokeDelegateWithConditionalAccessAnalyzer.cs

@@ -28,7 +28,7 @@ internal class InvokeDelegateWithConditionalAccessAnalyzer : DiagnosticAnalyzer,
             customTags: DiagnosticCustomTags.Unnecessary);
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(s_descriptor);
-        public bool RunInProcess => false;
+        public bool OpenFileOnly(Workspace workspace) => false;
 
         public override void Initialize(AnalysisContext context)
         {

src/Features/Core/Portable/Diagnostics/AnalyzerHelper.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
 using System;
+using System.Linq;
 using System.Reflection;
 using Microsoft.CodeAnalysis.ErrorReporting;
 using Microsoft.CodeAnalysis.Options;
@@ -44,20 +45,45 @@ public static bool IsBuiltInAnalyzer(this DiagnosticAnalyzer analyzer)
             return analyzer is IBuiltInAnalyzer || analyzer.IsWorkspaceDiagnosticAnalyzer() || analyzer.IsCompilerAnalyzer();
         }
 
-        public static bool MustRunInProcess(this DiagnosticAnalyzer analyzer)
+        public static bool ShouldRunForFullProject(this DiagnosticAnalyzerService service, DiagnosticAnalyzer analyzer, Project project)
         {
+            var builtInAnalyzer = analyzer as IBuiltInAnalyzer;
+            if (builtInAnalyzer != null)
+            {
+                return !builtInAnalyzer.OpenFileOnly(project.Solution.Workspace);
+            }
+
             if (analyzer.IsWorkspaceDiagnosticAnalyzer())
             {
                 return true;
             }
 
-            var builtInAnalyzer = analyzer as IBuiltInAnalyzer;
-            if (builtInAnalyzer == null)
+            // most of analyzers, number of descriptor is quite small, so this should be cheap.
+            return service.GetDiagnosticDescriptors(analyzer).Any(d => d.GetEffectiveSeverity(project.CompilationOptions) != ReportDiagnostic.Hidden);
+        }
+
+        public static ReportDiagnostic GetEffectiveSeverity(this DiagnosticDescriptor descriptor, CompilationOptions options)
+        {
+            return options == null
+                ? descriptor.DefaultSeverity.MapSeverityToReport()
+                : descriptor.GetEffectiveSeverity(options);
+        }
+
+        public static ReportDiagnostic MapSeverityToReport(this DiagnosticSeverity severity)
+        {
+            switch (severity)
             {
-                return false;
+                case DiagnosticSeverity.Hidden:
+                    return ReportDiagnostic.Hidden;
+                case DiagnosticSeverity.Info:
+                    return ReportDiagnostic.Info;
+                case DiagnosticSeverity.Warning:
+                    return ReportDiagnostic.Warn;
+                case DiagnosticSeverity.Error:
+                    return ReportDiagnostic.Error;
+                default:
+                    throw ExceptionUtilities.Unreachable;
             }
-
-            return builtInAnalyzer.RunInProcess;
         }
 
         public static bool IsCompilerAnalyzer(this DiagnosticAnalyzer analyzer)

src/Features/Core/Portable/Diagnostics/Analyzers/NamingStyles/NamingStyleDiagnosticAnalyzerBase.cs

@@ -37,7 +37,7 @@ internal abstract class NamingStyleDiagnosticAnalyzerBase : DiagnosticAnalyzer,
             }.ToImmutableArray();
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(s_descriptorNamingStyle);
-        public bool RunInProcess => true;
+        public bool OpenFileOnly(Workspace workspace) => true;
 
         public override void Initialize(AnalysisContext context)
         {

src/Features/Core/Portable/Diagnostics/Analyzers/PreferFrameworkTypeDiagnosticAnalyzerBase.cs

@@ -45,7 +45,7 @@ internal abstract class PreferFrameworkTypeDiagnosticAnalyzerBase<TSyntaxKind, T
         private PerLanguageOption<CodeStyleOption<bool>> GetOptionForMemberAccessContext =>
             CodeStyleOptions.PreferIntrinsicPredefinedTypeKeywordInMemberAccess;
 
-        public bool RunInProcess => true;
+        public bool OpenFileOnly(Workspace workspace) => true;
 
         public DiagnosticAnalyzerCategory GetAnalyzerCategory() => DiagnosticAnalyzerCategory.SemanticSpanAnalysis;
 

src/Features/Core/Portable/Diagnostics/Analyzers/QualifyMemberAccessDiagnosticAnalyzerBase.cs

@@ -26,7 +26,7 @@ internal abstract class QualifyMemberAccessDiagnosticAnalyzerBase<TLanguageKindE
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(s_descriptorQualifyMemberAccess);
 
-        public bool RunInProcess => true;
+        public bool OpenFileOnly(Workspace workspace) => true;
 
         protected abstract bool IsAlreadyQualifiedMemberAccess(SyntaxNode node);
 

src/Features/Core/Portable/Diagnostics/Analyzers/RemoveUnnecessaryCastDiagnosticAnalyzerBase.cs

@@ -24,7 +24,7 @@ internal abstract class RemoveUnnecessaryCastDiagnosticAnalyzerBase<TLanguageKin
         #region Interface methods
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(s_descriptor);
-        public bool RunInProcess => false;
+        public bool OpenFileOnly(Workspace workspace) => false;
 
         public override void Initialize(AnalysisContext context)
         {

src/Features/Core/Portable/Diagnostics/Analyzers/RemoveUnnecessaryImportsDiagnosticAnalyzerBase.cs

@@ -49,7 +49,7 @@ private DiagnosticDescriptor GetClassificationIdDescriptor()
         }
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(s_fixableIdDescriptor, GetClassificationIdDescriptor());
-        public bool RunInProcess => true;
+        public bool OpenFileOnly(Workspace workspace) => true;
 
         public override void Initialize(AnalysisContext context)
         {

src/Features/Core/Portable/Diagnostics/Analyzers/RudeEditUserDiagnosticAnalyzer.cs

@@ -14,7 +14,7 @@ namespace Microsoft.CodeAnalysis.EditAndContinue
     internal class RudeEditDiagnosticAnalyzer : DocumentDiagnosticAnalyzer, IBuiltInAnalyzer
     {
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => RudeEditDiagnosticDescriptors.AllDescriptors;
-        public bool RunInProcess => true;
+        public bool OpenFileOnly(Workspace workspace) => false;
 
         public override Task<ImmutableArray<Diagnostic>> AnalyzeSyntaxAsync(Document document, CancellationToken cancellationToken)
         {
@@ -55,6 +55,7 @@ public override async Task<ImmutableArray<Diagnostic>> AnalyzeSemanticsAsync(Doc
                 throw ExceptionUtilities.Unreachable;
             }
         }
+
         public DiagnosticAnalyzerCategory GetAnalyzerCategory()
         {
             return DiagnosticAnalyzerCategory.SemanticDocumentAnalysis;

src/Features/Core/Portable/Diagnostics/Analyzers/SimplifyTypeNamesDiagnosticAnalyzerBase.cs

@@ -72,7 +72,7 @@ public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics
             }
         }
 
-        public bool RunInProcess => true;
+        public bool OpenFileOnly(Workspace workspace) => true;
 
         protected abstract void AnalyzeNode(SyntaxNodeAnalysisContext context);
 

src/Features/Core/Portable/Diagnostics/Analyzers/UnboundIdentifiersDiagnosticAnalyzerBase.cs

@@ -20,7 +20,7 @@ internal abstract class UnboundIdentifiersDiagnosticAnalyzerBase<TLanguageKindEn
         protected abstract bool ConstructorDoesNotExist(SyntaxNode node, SymbolInfo info, SemanticModel semanticModel);
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(DiagnosticDescriptor, DiagnosticDescriptor2);
-        public bool RunInProcess => false;
+        public bool OpenFileOnly(Workspace workspace) => false;
 
         public override void Initialize(AnalysisContext context)
         {

src/Features/Core/Portable/Diagnostics/BaseDiagnosticIncrementalAnalyzer.cs

@@ -234,29 +234,5 @@ public virtual void Shutdown()
         {
             return Owner.GetOnAnalyzerException(projectId, DiagnosticLogAggregator);
         }
-
-        protected static ReportDiagnostic GetEffectiveSeverity(DiagnosticDescriptor descriptor, CompilationOptions options)
-        {
-            return options == null
-                ? MapSeverityToReport(descriptor.DefaultSeverity)
-                : descriptor.GetEffectiveSeverity(options);
-        }
-
-        protected static ReportDiagnostic MapSeverityToReport(DiagnosticSeverity severity)
-        {
-            switch (severity)
-            {
-                case DiagnosticSeverity.Hidden:
-                    return ReportDiagnostic.Hidden;
-                case DiagnosticSeverity.Info:
-                    return ReportDiagnostic.Info;
-                case DiagnosticSeverity.Warning:
-                    return ReportDiagnostic.Warn;
-                case DiagnosticSeverity.Error:
-                    return ReportDiagnostic.Error;
-                default:
-                    throw ExceptionUtilities.Unreachable;
-            }
-        }
     }
 }

src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer_IncrementalAnalyzer.cs

@@ -252,8 +252,7 @@ private bool ShouldRunForFullProject(DiagnosticAnalyzer analyzer, Project projec
                 return true;
             }
 
-            // most of analyzers, number of descriptor is quite small, so this should be cheap.
-            return Owner.GetDiagnosticDescriptors(analyzer).Any(d => GetEffectiveSeverity(d, project.CompilationOptions) != ReportDiagnostic.Hidden);
+            return Owner.ShouldRunForFullProject(analyzer, project);
         }
 
         private void RaiseProjectDiagnosticsIfNeeded(

src/Features/Core/Portable/Diagnostics/IBuiltInAnalyzer.cs

@@ -14,14 +14,10 @@ internal interface IBuiltInAnalyzer
         DiagnosticAnalyzerCategory GetAnalyzerCategory();
 
         /// <summary>
-        /// This indicates whether this builtin analyzer must run in proc or can be run on remote host such as service hub.
+        /// This indicates whether this builtin analyzer will only run on opened files.
         /// 
-        /// if the diagnostic analyzer can run in command line as it is, then it should be able to run in remote host. 
-        /// otherwise, it won't unless diagnostic analyzer author make changes in remote host to provide whatever missing
-        /// data command line build doesn't provide such as workspace options/services/MEF and etc.
-        /// 
-        /// at this moment, remote host provide same context as command line build and only that context
+        /// all analyzers that want to run on closed files must be able to run in remote host.
         /// </summary>
-        bool RunInProcess { get; }
+        bool OpenFileOnly(Workspace workspace);
     }
 }

src/Features/Core/Portable/PopulateSwitch/PopulateSwitchDiagnosticAnalyzer.cs

@@ -26,7 +26,7 @@ internal sealed class PopulateSwitchDiagnosticAnalyzer : DiagnosticAnalyzer, IBu
         #region Interface methods
 
         public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(s_descriptor);
-        public bool RunInProcess => false;
+        public bool OpenFileOnly(Workspace workspace) => false;
 
         public override void Initialize(AnalysisContext context)
         {

src/VisualStudio/Next/Diagnostics/OutOfProcDiagnosticAnalyzerExecutor.cs

@@ -44,47 +44,26 @@ internal class OutOfProcDiagnosticAnalyzerExecutor : IRemoteHostDiagnosticAnalyz
             if (remoteHostClient == null)
             {
                 // remote host is not running. this can happen if remote host is disabled.
-                return await AnalyzeInProcAsync(analyzerDriver, project, cancellationToken).ConfigureAwait(false);
+                return await InProcCodeAnalysisDiagnosticAnalyzerExecutor.Instance.AnalyzeAsync(analyzerDriver, project, cancellationToken).ConfigureAwait(false);
             }
 
-            // TODO: later, make sure we can run all analyzer on remote host. 
-            //       for now, we will check whether built in analyzer can run on remote host and only those run on remote host.
-            var inProcResultTask = AnalyzeInProcAsync(CreateAnalyzerDriver(analyzerDriver, a => a.MustRunInProcess()), project, cancellationToken);
-            var outOfProcResultTask = AnalyzeOutOfProcAsync(remoteHostClient, analyzerDriver, project, cancellationToken);
-
-            // run them concurrently in vs and remote host
-            await Task.WhenAll(inProcResultTask, outOfProcResultTask).ConfigureAwait(false);
+            var outOfProcResult = await AnalyzeOutOfProcAsync(remoteHostClient, analyzerDriver, project, cancellationToken).ConfigureAwait(false);
 
             // make sure things are not cancelled
             cancellationToken.ThrowIfCancellationRequested();
 
-            // merge 2 results
-            return DiagnosticAnalysisResultMap.Create(
-                inProcResultTask.Result.AnalysisResult.AddRange(outOfProcResultTask.Result.AnalysisResult),
-                inProcResultTask.Result.TelemetryInfo.AddRange(outOfProcResultTask.Result.TelemetryInfo));
-        }
-
-        private async Task<DiagnosticAnalysisResultMap<DiagnosticAnalyzer, DiagnosticAnalysisResult>> AnalyzeInProcAsync(CompilationWithAnalyzers analyzerDriver, Project project, CancellationToken cancellationToken)
-        {
-            if (analyzerDriver == null)
-            {
-                // no analyzers for in proc process
-                return DiagnosticAnalysisResultMap.Create(ImmutableDictionary<DiagnosticAnalyzer, DiagnosticAnalysisResult>.Empty, ImmutableDictionary<DiagnosticAnalyzer, AnalyzerTelemetryInfo>.Empty);
-            }
-
-            return await InProcCodeAnalysisDiagnosticAnalyzerExecutor.Instance.AnalyzeAsync(analyzerDriver, project, cancellationToken).ConfigureAwait(false);
+            return DiagnosticAnalysisResultMap.Create(outOfProcResult.AnalysisResult, outOfProcResult.TelemetryInfo);
         }
 
         private async Task<DiagnosticAnalysisResultMap<DiagnosticAnalyzer, DiagnosticAnalysisResult>> AnalyzeOutOfProcAsync(
             RemoteHostClient client, CompilationWithAnalyzers analyzerDriver, Project project, CancellationToken cancellationToken)
         {
             var solution = project.Solution;
-
             var snapshotService = solution.Workspace.Services.GetService<ISolutionChecksumService>();
 
             // TODO: this should be moved out
             var hostChecksums = GetHostAnalyzerReferences(snapshotService, _analyzerService.GetHostAnalyzerReferences(), cancellationToken);
-            var analyzerMap = CreateAnalyzerMap(analyzerDriver.Analyzers.Where(a => !a.MustRunInProcess()));
+            var analyzerMap = CreateAnalyzerMap(analyzerDriver.Analyzers);
             if (analyzerMap.Count == 0)
             {
                 return DiagnosticAnalysisResultMap.Create(ImmutableDictionary<DiagnosticAnalyzer, DiagnosticAnalysisResult>.Empty, ImmutableDictionary<DiagnosticAnalyzer, AnalyzerTelemetryInfo>.Empty);