Merge pull request #36113 from sharwell/incremental-analyzers

Update AnalyzerRunner to support IIncrementalAnalyzer testing

eng/Versions.props

@@ -39,7 +39,7 @@
     <ICSharpCodeDecompilerVersion>4.0.0.4285-beta1</ICSharpCodeDecompilerVersion>
     <MicrosoftBuildVersion>15.1.548</MicrosoftBuildVersion>
     <MicrosoftBuildFrameworkVersion>15.1.548</MicrosoftBuildFrameworkVersion>
-    <MicrosoftBuildLocatorVersion>1.0.13</MicrosoftBuildLocatorVersion>
+    <MicrosoftBuildLocatorVersion>1.2.2</MicrosoftBuildLocatorVersion>
     <MicrosoftBuildRuntimeVersion>15.1.548</MicrosoftBuildRuntimeVersion>
     <MicrosoftBuildTasksCoreVersion>15.1.548</MicrosoftBuildTasksCoreVersion>
     <MicrosoftCodeAnalysisAnalyzersVersion>$(RoslynDiagnosticsNugetPackageVersion)</MicrosoftCodeAnalysisAnalyzersVersion>
@@ -163,6 +163,7 @@
     <RoslynOptProfRunSettingsGeneratorVersion>1.0.0-beta3.19057.1</RoslynOptProfRunSettingsGeneratorVersion>
     <RoslynToolsLightUpSystemRuntimeLoaderFixedVersion>4.3.0</RoslynToolsLightUpSystemRuntimeLoaderFixedVersion>
     <RoslynMicrosoftVisualStudioExtensionManagerVersion>0.0.4</RoslynMicrosoftVisualStudioExtensionManagerVersion>
+    <SourceBrowserVersion>1.0.21</SourceBrowserVersion>
     <StreamJsonRpcVersion>2.0.167</StreamJsonRpcVersion>
     <SystemCollectionsImmutableVersion>1.5.0</SystemCollectionsImmutableVersion>
     <SystemCommandLineExperimentalVersion>0.1.0-alpha-63729-01</SystemCommandLineExperimentalVersion>

eng/targets/Settings.props

@@ -30,7 +30,7 @@
     <UseRoslynAnalyzers Condition="'$(UseRoslynAnalyzers)' == ''">true</UseRoslynAnalyzers>
 
     <!-- Set to non-existent file to prevent common targets from importing Microsoft.CodeAnalysis.targets -->
-    <CodeAnalysisTargets>*none*</CodeAnalysisTargets>
+    <CodeAnalysisTargets>NON_EXISTENT_FILE</CodeAnalysisTargets>
 
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">15.0</VisualStudioVersion>
     <VisualStudioReferenceMajorVersion Condition="'$(VisualStudioReferenceMajorVersion)' == ''">$(VisualStudioVersion.Substring($([System.Convert]::ToInt32(0)), $(VisualStudioVersion.IndexOf('.'))))</VisualStudioReferenceMajorVersion>

src/Tools/AnalyzerRunner/AnalyzerRunner.csproj

@@ -8,9 +8,35 @@
     <!-- Automatically generate the necessary assembly binding redirects -->
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
   </PropertyGroup>
+
+  <Choose>
+    <When Condition="'$(OS)' == 'Windows_NT'">
+      <PropertyGroup>
+        <RuntimeIdentifier>win-x64</RuntimeIdentifier>
+      </PropertyGroup>
+    </When>
+    <When Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">
+      <PropertyGroup>
+        <RuntimeIdentifier>osx-x64</RuntimeIdentifier>
+      </PropertyGroup>
+    </When>
+    <When Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">
+      <PropertyGroup>
+        <RuntimeIdentifier>linux-x64</RuntimeIdentifier>
+      </PropertyGroup>
+    </When>
+  </Choose>
+
   <ItemGroup>
+    <Reference Include="System.ComponentModel.Composition" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="SQLitePCLRaw.bundle_green" Version="$(SQLitePCLRawbundle_greenVersion)" PrivateAssets="all" />
     <PackageReference Include="Microsoft.Build.Locator" Version="$(MicrosoftBuildLocatorVersion)" />
+    <PackageReference Include="SourceBrowser" Version="$(SourceBrowserVersion)" ExcludeAssets="build;compile" PrivateAssets="all" />
   </ItemGroup>
+
   <ItemGroup Label="Project References">
     <ProjectReference Include="..\..\Compilers\Core\Portable\Microsoft.CodeAnalysis.csproj" />
     <ProjectReference Include="..\..\Compilers\CSharp\Portable\Microsoft.CodeAnalysis.CSharp.csproj" />
@@ -29,6 +55,12 @@
     <ProjectReference Include="..\..\Features\CSharp\Portable\Microsoft.CodeAnalysis.CSharp.Features.csproj" />
     <ProjectReference Include="..\..\Features\VisualBasic\Portable\Microsoft.CodeAnalysis.VisualBasic.Features.vbproj" />
   </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="$(NuGetPackageRoot)\sourcebrowser\$(SourceBrowserVersion)\tools\HtmlGenerator.exe" Visible="false" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="$(NuGetPackageRoot)\sourcebrowser\$(SourceBrowserVersion)\tools\Microsoft.SourceBrowser.Common.dll" Visible="false" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+
   <ItemGroup>
     <None Include="app.config" />
   </ItemGroup>

src/Tools/AnalyzerRunner/AnalyzerRunnerMefHostServices.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System.Collections.Immutable;
+using System.Reflection;
+using System.Threading;
+using Microsoft.CodeAnalysis.Host.Mef;
+
+namespace AnalyzerRunner
+{
+    internal static class AnalyzerRunnerMefHostServices
+    {
+        private static MefHostServices s_defaultServices;
+
+        public static MefHostServices DefaultServices
+        {
+            get
+            {
+                if (s_defaultServices is null)
+                {
+                    Interlocked.CompareExchange(ref s_defaultServices, MefHostServices.Create(DefaultAssemblies), null);
+                }
+
+                return s_defaultServices;
+            }
+        }
+
+        private static ImmutableArray<Assembly> DefaultAssemblies
+            => MSBuildMefHostServices.DefaultAssemblies.Add(typeof(AnalyzerRunnerMefHostServices).Assembly);
+    }
+}

src/Tools/AnalyzerRunner/IncrementalAnalyzerRunner.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Runtime;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.FindSymbols.SymbolTree;
+using Microsoft.CodeAnalysis.Host;
+using Microsoft.CodeAnalysis.Host.Mef;
+using Microsoft.CodeAnalysis.IncrementalCaches;
+using Microsoft.CodeAnalysis.Shared.Options;
+using Microsoft.CodeAnalysis.SolutionCrawler;
+using Microsoft.CodeAnalysis.SolutionSize;
+using Microsoft.CodeAnalysis.Storage;
+
+namespace AnalyzerRunner
+{
+    internal class IncrementalAnalyzerRunner
+    {
+        private readonly Options _options;
+
+        public IncrementalAnalyzerRunner(Options options)
+        {
+            _options = options;
+        }
+
+        public bool HasAnalyzers => _options.IncrementalAnalyzerNames.Any();
+
+        internal async Task RunAsync(Workspace workspace, CancellationToken cancellationToken)
+        {
+            if (!HasAnalyzers)
+            {
+                return;
+            }
+
+            workspace.Options = workspace.Options
+                .WithChangedOption(StorageOptions.SolutionSizeThreshold, _options.UsePersistentStorage ? 0 : int.MaxValue)
+                .WithChangedOption(RuntimeOptions.FullSolutionAnalysis, _options.FullSolutionAnalysis)
+                .WithChangedOption(ServiceFeatureOnOffOptions.ClosedFileDiagnostic, LanguageNames.CSharp, _options.FullSolutionAnalysis)
+                .WithChangedOption(ServiceFeatureOnOffOptions.ClosedFileDiagnostic, LanguageNames.VisualBasic, _options.FullSolutionAnalysis);
+
+            if (!string.IsNullOrEmpty(_options.ProfileRoot))
+            {
+                ProfileOptimization.StartProfile(nameof(IIncrementalAnalyzer));
+            }
+
+            var exportProvider = (IMefHostExportProvider)workspace.Services.HostServices;
+
+            var solutionSizeTracker = exportProvider.GetExports<ISolutionSizeTracker>().Single().Value;
+
+            // This will return the tracker, since it's a singleton.
+            var analyzer = ((IIncrementalAnalyzerProvider)solutionSizeTracker).CreateIncrementalAnalyzer(workspace);
+            await analyzer.NewSolutionSnapshotAsync(workspace.CurrentSolution, cancellationToken).ConfigureAwait(false);
+
+            var solutionCrawlerRegistrationService = (SolutionCrawlerRegistrationService)workspace.Services.GetRequiredService<ISolutionCrawlerRegistrationService>();
+            solutionCrawlerRegistrationService.Register(workspace);
+            solutionCrawlerRegistrationService.WaitUntilCompletion_ForTestingPurposesOnly(workspace, ImmutableArray.Create(analyzer));
+
+            var size = solutionSizeTracker.GetSolutionSize(workspace, workspace.CurrentSolution.Id);
+            Console.WriteLine("Current solution size:\t" + size);
+
+            if (_options.UsePersistentStorage)
+            {
+                if (size <= 0)
+                {
+                    throw new InvalidOperationException("Solution size is too small; persistent storage is disabled.");
+                }
+
+                var persistentStorageService = workspace.Services.GetRequiredService<IPersistentStorageService>();
+                var persistentStorage = persistentStorageService.GetStorage(workspace.CurrentSolution);
+                if (persistentStorage is NoOpPersistentStorage)
+                {
+                    throw new InvalidOperationException("Benchmark is not configured to use persistent storage.");
+                }
+            }
+
+            var incrementalAnalyzerProviders = exportProvider.GetExports<IIncrementalAnalyzerProvider, IncrementalAnalyzerProviderMetadata>();
+
+            foreach (var incrementalAnalyzerName in _options.IncrementalAnalyzerNames)
+            {
+                var incrementalAnalyzerProvider = incrementalAnalyzerProviders.Where(x => x.Metadata.Name == incrementalAnalyzerName).SingleOrDefault(provider => provider.Metadata.WorkspaceKinds?.Contains(workspace.Kind) ?? false)?.Value;
+                incrementalAnalyzerProvider ??= incrementalAnalyzerProviders.Where(x => x.Metadata.Name == incrementalAnalyzerName).SingleOrDefault(provider => provider.Metadata.WorkspaceKinds?.Contains(WorkspaceKind.Host) ?? false)?.Value;
+                incrementalAnalyzerProvider ??= incrementalAnalyzerProviders.Where(x => x.Metadata.Name == incrementalAnalyzerName).Single(provider => provider.Metadata.WorkspaceKinds is null).Value;
+                var incrementalAnalyzer = incrementalAnalyzerProvider.CreateIncrementalAnalyzer(workspace);
+                solutionCrawlerRegistrationService.WaitUntilCompletion_ForTestingPurposesOnly(workspace, ImmutableArray.Create(incrementalAnalyzer));
+
+                switch (incrementalAnalyzerName)
+                {
+                    case nameof(SymbolTreeInfoIncrementalAnalyzerProvider):
+                        var symbolTreeInfoCacheService = workspace.Services.GetRequiredService<ISymbolTreeInfoCacheService>();
+                        var symbolTreeInfo = await symbolTreeInfoCacheService.TryGetSourceSymbolTreeInfoAsync(workspace.CurrentSolution.Projects.First(), cancellationToken).ConfigureAwait(false);
+                        if (symbolTreeInfo is null)
+                        {
+                            throw new InvalidOperationException("Benchmark failed to calculate symbol tree info.");
+                        }
+
+                        break;
+
+                    default:
+                        // No additional actions required
+                        break;
+                }
+            }
+        }
+    }
+}

src/Tools/AnalyzerRunner/Options.cs

@@ -7,7 +7,7 @@
 
 namespace AnalyzerRunner
 {
-    internal class Options
+    internal sealed class Options
     {
         public readonly string AnalyzerPath;
         public readonly string SolutionPath;
@@ -22,8 +22,14 @@ internal class Options
         public readonly Func<string, bool> TestDocumentMatch;
         public readonly int TestDocumentIterations;
         public readonly string LogFileName;
+        public readonly string ProfileRoot;
 
-        public Options(
+        // Options specific to incremental analyzers
+        public readonly bool UsePersistentStorage;
+        public readonly bool FullSolutionAnalysis;
+        public readonly ImmutableList<string> IncrementalAnalyzerNames;
+
+        private Options(
             string analyzerPath,
             string solutionPath,
             ImmutableHashSet<string> analyzerIds,
@@ -35,7 +41,11 @@ internal class Options
             bool testDocuments,
             Func<string, bool> testDocumentMatch,
             int testDocumentIterations,
-            string logFileName)
+            string logFileName,
+            string profileRoot,
+            bool usePersistentStorage,
+            bool fullSolutionAnalysis,
+            ImmutableList<string> incrementalAnalyzerNames)
         {
             AnalyzerPath = analyzerPath;
             SolutionPath = solutionPath;
@@ -49,6 +59,10 @@ internal class Options
             TestDocumentMatch = testDocumentMatch;
             TestDocumentIterations = testDocumentIterations;
             LogFileName = logFileName;
+            ProfileRoot = profileRoot;
+            UsePersistentStorage = usePersistentStorage;
+            FullSolutionAnalysis = fullSolutionAnalysis;
+            IncrementalAnalyzerNames = incrementalAnalyzerNames;
         }
 
         internal static Options Create(string[] args)
@@ -65,6 +79,10 @@ internal static Options Create(string[] args)
             Func<string, bool> testDocumentMatch = _ => true;
             int testDocumentIterations = 10;
             string logFileName = null;
+            string profileRoot = null;
+            var usePersistentStorage = false;
+            var fullSolutionAnalysis = false;
+            var incrementalAnalyzerNames = ImmutableList.CreateBuilder<string>();
 
             int i = 0;
             while (i < args.Length)
@@ -106,6 +124,18 @@ internal static Options Create(string[] args)
                     case "/log":
                         logFileName = ReadValue();
                         break;
+                    case "/profileroot":
+                        profileRoot = ReadValue();
+                        break;
+                    case "/persist":
+                        usePersistentStorage = true;
+                        break;
+                    case "/fsa":
+                        fullSolutionAnalysis = true;
+                        break;
+                    case "/ia":
+                        incrementalAnalyzerNames.Add(ReadValue());
+                        break;
                     default:
                         if (analyzerPath == null)
                         {
@@ -147,7 +177,11 @@ internal static Options Create(string[] args)
                 testDocuments: testDocuments,
                 testDocumentMatch: testDocumentMatch,
                 testDocumentIterations: testDocumentIterations,
-                logFileName: logFileName);
+                logFileName: logFileName,
+                profileRoot: profileRoot,
+                usePersistentStorage: usePersistentStorage,
+                fullSolutionAnalysis: fullSolutionAnalysis,
+                incrementalAnalyzerNames: incrementalAnalyzerNames.ToImmutable());
         }
     }
 }

src/Tools/AnalyzerRunner/PersistentStorageLocationService.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Composition;
+using System.IO;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Host;
+using Microsoft.CodeAnalysis.Host.Mef;
+
+namespace AnalyzerRunner
+{
+    [ExportWorkspaceService(typeof(IPersistentStorageLocationService), ServiceLayer.Host)]
+    [Shared]
+    internal class PersistentStorageLocationService : IPersistentStorageLocationService
+    {
+        public event EventHandler<PersistentStorageLocationChangingEventArgs> StorageLocationChanging
+        {
+            add { }
+            remove { }
+        }
+
+        public bool IsSupported(Workspace workspace) => true;
+
+        public string TryGetStorageLocation(SolutionId solutionId)
+        {
+            var location = Path.Combine(Path.GetTempPath(), "RoslynTests", "AnalyzerRunner", "temp-db");
+            Directory.CreateDirectory(location);
+            return location;
+        }
+    }
+}

src/Tools/AnalyzerRunner/Properties/launchSettings.json

@@ -2,23 +2,23 @@
   "profiles": {
     "IDE Analyzers": {
       "commandName": "Project",
-      "commandLineArgs": "$(SolutionDir)artifacts\\bin\\AnalyzerRunner\\$(Configuration)\\net472\\ $(SolutionDir)Roslyn.sln /concurrent /stats"
+      "commandLineArgs": "$(OutDir) $(SolutionDir)Roslyn.sln /concurrent /stats"
     },
     "CSharpEditorFeatures Analyzers": {
       "commandName": "Project",
-      "commandLineArgs": "$(SolutionDir)artifacts\\bin\\AnalyzerRunner\\$(Configuration)\\net472\\Microsoft.CodeAnalysis.CSharp.EditorFeatures.dll $(SolutionDir)Roslyn.sln /concurrent /stats"
+      "commandLineArgs": "$(OutDir)Microsoft.CodeAnalysis.CSharp.EditorFeatures.dll $(SolutionDir)Roslyn.sln /concurrent /stats"
     },
     "CSharpFeatures Analyzers": {
       "commandName": "Project",
-      "commandLineArgs": "$(SolutionDir)artifacts\\bin\\AnalyzerRunner\\$(Configuration)\\net472\\Microsoft.CodeAnalysis.CSharp.Features.dll $(SolutionDir)Roslyn.sln /concurrent /stats"
+      "commandLineArgs": "$(OutDir)Microsoft.CodeAnalysis.CSharp.Features.dll $(SolutionDir)Roslyn.sln /concurrent /stats"
     },
     "BasicEditorFeatures Analyzers": {
       "commandName": "Project",
-      "commandLineArgs": "$(SolutionDir)artifacts\\bin\\AnalyzerRunner\\$(Configuration)\\net472\\Microsoft.CodeAnalysis.VisualBasic.EditorFeatures.dll $(SolutionDir)Roslyn.sln /concurrent /stats"
+      "commandLineArgs": "$(OutDir)Microsoft.CodeAnalysis.VisualBasic.EditorFeatures.dll $(SolutionDir)Roslyn.sln /concurrent /stats"
     },
     "BasicFeatures Analyzers": {
       "commandName": "Project",
-      "commandLineArgs": "$(SolutionDir)artifacts\\bin\\AnalyzerRunner\\$(Configuration)\\net472\\Microsoft.CodeAnalysis.VisualBasic.Features.dll $(SolutionDir)Roslyn.sln /concurrent /stats"
+      "commandLineArgs": "$(OutDir)Microsoft.CodeAnalysis.VisualBasic.Features.dll $(SolutionDir)Roslyn.sln /concurrent /stats"
     },
     "Microsoft.CodeQuality.CSharp.Analyzers Analyzers": {
       "commandName": "Project",
@@ -26,7 +26,15 @@
     },
     "Profile CSharpSimplifyTypeNamesDiagnosticAnalyzer": {
       "commandName": "Project",
-      "commandLineArgs": "$(SolutionDir)artifacts\\bin\\AnalyzerRunner\\$(Configuration)\\net472\\Microsoft.CodeAnalysis.CSharp.Features.dll $(SolutionDir)..\\Orchard\\src\\Orchard.sln /stats /a CSharpSimplifyTypeNamesDiagnosticAnalyzer /editperf:ContentQueryExtensions /edititer:300"
+      "commandLineArgs": "$(OutDir)Microsoft.CodeAnalysis.CSharp.Features.dll $(SolutionDir)..\\Orchard\\src\\Orchard.sln /stats /a CSharpSimplifyTypeNamesDiagnosticAnalyzer /editperf:ContentQueryExtensions /edititer:300"
+    },
+    "IIncrementalAnalyzer SymbolTreeInfoIncrementalAnalyzerProvider": {
+      "commandName": "Project",
+      "commandLineArgs": "$(OutDir) $(SolutionDir)Roslyn.sln /stats /ia SymbolTreeInfoIncrementalAnalyzerProvider /persist /profileroot $(SolutionDir)artifacts\\profileRoot"
+    },
+    "IIncrementalAnalyzer DiagnosticAnalyzerService": {
+      "commandName": "Project",
+      "commandLineArgs": "$(OutDir) $(SolutionDir)Roslyn.sln /stats /ia Diagnostic /persist /profileroot $(SolutionDir)artifacts\\profileRoot"
     }
   }
 }

src/Tools/AnalyzerRunner/app.config

@@ -2,5 +2,7 @@
 <configuration>
   <runtime>
     <gcServer enabled="true" />
+    <appDomainManagerType value="Microsoft.SourceBrowser.Common.CustomAppDomainManager"/>
+    <appDomainManagerAssembly value="Microsoft.SourceBrowser.Common"/>
   </runtime>
 </configuration>

src/Workspaces/Core/Portable/Microsoft.CodeAnalysis.Workspaces.csproj

@@ -224,6 +224,7 @@
     </Compile>
   </ItemGroup>
   <ItemGroup>
+    <InternalsVisibleTo Include="AnalyzerRunner" />
     <InternalsVisibleTo Include="csi" />
     <InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.EditorFeatures" />
     <InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.Features" />