Merge pull request #1395 from mavasani/FixAnalyzerMemoryLeaks

Fix memory leaks in AnalyzerManager: 

1) Statically created LocalizableString instances by analyzers were holding onto instances of AnalyzerExecutor (which holds onto the compilation on which it executes) for exception reporting, causing us to leak compilations in command line builds. Fixed this by making sure that we unregister these exception handlers during analyzer cleanup in CommonCompiler, we already did so for VisualStudioAnalyzer created in IDE.

2) Switch all the state caches in AnalyzerManager to be keyed with analyzer, and clear all state for analyzer when host disposes it. Performance analysis showed that analyzers that capture the CompilationStartAnalysisContext in its RegisterCompilationStartAction via some lambda were rooting the compilation objects.

Above two changes got rid of all the static and dependent handles rooting compilations during command line builds, and I see a perceived reduction in memory used by VBCSCompiler during building Roslyn.

3) Fix the IDE onAnalyzerException delegate to not capture project instance, but instead use the projectId. Otherwise, VSIX analyzers that live for lifetime of VS instance would leak compilations.

While I was at it, I also got rid of functionality added to MetadataCache that was caching and re-using analyzer instances across AnalyzerFileReference instances, we had already decided to instead keep lifetime of analyzer instances bound by lifetime of owning AnalyzerFileReference.

src/Compilers/Core/AnalyzerDriver/AnalyzerManager.AnalyzerAndOptions.cs

@@ -9,7 +9,7 @@ internal partial class AnalyzerManager
     {
         private sealed class AnalyzerAndOptions
         {
-            private readonly DiagnosticAnalyzer _analyzer;
+            public readonly DiagnosticAnalyzer Analyzer;
             private readonly AnalyzerOptions _analyzerOptions;
 
             public AnalyzerAndOptions(DiagnosticAnalyzer analyzer, AnalyzerOptions analyzerOptions)
@@ -17,7 +17,7 @@ public AnalyzerAndOptions(DiagnosticAnalyzer analyzer, AnalyzerOptions analyzerO
                 Debug.Assert(analyzer != null);
                 Debug.Assert(analyzerOptions != null);
 
-                _analyzer = analyzer;
+                Analyzer = analyzer;
                 _analyzerOptions = analyzerOptions;
             }
 
@@ -29,7 +29,7 @@ public bool Equals(AnalyzerAndOptions other)
                 }
 
                 return other != null &&
-                    _analyzer.Equals(other._analyzer) &&
+                    Analyzer.Equals(other.Analyzer) &&
                     _analyzerOptions.Equals(other._analyzerOptions);
             }
 
@@ -40,7 +40,7 @@ public override bool Equals(object other)
 
             public override int GetHashCode()
             {
-                return Hash.Combine(_analyzer.GetHashCode(), _analyzerOptions.GetHashCode());
+                return Hash.Combine(Analyzer.GetHashCode(), _analyzerOptions.GetHashCode());
             }
         }
     }

src/Compilers/Core/AnalyzerDriver/AnalyzerManager.cs

@@ -4,6 +4,7 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 
@@ -31,31 +32,28 @@ internal partial class AnalyzerManager
 
         // This map stores the tasks to compute HostSessionStartAnalysisScope for session wide analyzer actions, i.e. AnalyzerActions registered by analyzer's Initialize method.
         // These are run only once per every analyzer.
-        private readonly ConditionalWeakTable<DiagnosticAnalyzer, Task<HostSessionStartAnalysisScope>> _sessionScopeMap =
-            new ConditionalWeakTable<DiagnosticAnalyzer, Task<HostSessionStartAnalysisScope>>();
+        private readonly ConcurrentDictionary<DiagnosticAnalyzer, Task<HostSessionStartAnalysisScope>> _sessionScopeMap =
+            new ConcurrentDictionary<DiagnosticAnalyzer, Task<HostSessionStartAnalysisScope>>(concurrencyLevel: 2, capacity: 5);
 
         // This map stores the tasks to compute HostCompilationStartAnalysisScope for per-compilation analyzer actions, i.e. AnalyzerActions registered by analyzer's CompilationStartActions.
         // Compilation start actions will get executed once per-each AnalyzerAndOptions as user might want to return different set of custom actions for each compilation/analyzer options.
-        private readonly ConditionalWeakTable<Compilation, ConcurrentDictionary<AnalyzerAndOptions, Task<HostCompilationStartAnalysisScope>>> _compilationScopeMap =
-            new ConditionalWeakTable<Compilation, ConcurrentDictionary<AnalyzerAndOptions, Task<HostCompilationStartAnalysisScope>>>();
-
-        private readonly ConditionalWeakTable<Compilation, ConcurrentDictionary<AnalyzerAndOptions, Task<HostCompilationStartAnalysisScope>>>.CreateValueCallback _compilationScopeMapCallback =
-            comp => new ConcurrentDictionary<AnalyzerAndOptions, Task<HostCompilationStartAnalysisScope>>(concurrencyLevel: 2, capacity: 5);
+        private readonly ConcurrentDictionary<AnalyzerAndOptions, ConditionalWeakTable<Compilation, Task<HostCompilationStartAnalysisScope>>> _compilationScopeMap =
+            new ConcurrentDictionary<AnalyzerAndOptions, ConditionalWeakTable<Compilation, Task<HostCompilationStartAnalysisScope>>>(concurrencyLevel: 2, capacity: 5);
 
         /// <summary>
         /// Cache descriptors for each diagnostic analyzer. We do this since <see cref="DiagnosticAnalyzer.SupportedDiagnostics"/> is
         /// a property rather than metadata. We expect it to be cheap and immutable, but we can't force them to be so, we cache them
         /// and ask only once.
         /// </summary>
-        private readonly ConditionalWeakTable<DiagnosticAnalyzer, Tuple<ImmutableArray<DiagnosticDescriptor>, EventHandler<Exception>>> _descriptorCache =
-            new ConditionalWeakTable<DiagnosticAnalyzer, Tuple<ImmutableArray<DiagnosticDescriptor>, EventHandler<Exception>>>();
+        private readonly ConcurrentDictionary<DiagnosticAnalyzer, Tuple<ImmutableArray<DiagnosticDescriptor>, EventHandler<Exception>>> _descriptorCache =
+            new ConcurrentDictionary<DiagnosticAnalyzer, Tuple<ImmutableArray<DiagnosticDescriptor>, EventHandler<Exception>>>(concurrencyLevel: 2, capacity: 5);
 
         private Task<HostCompilationStartAnalysisScope> GetCompilationAnalysisScopeCoreAsync(
             AnalyzerAndOptions analyzerAndOptions,
             HostSessionStartAnalysisScope sessionScope,
             AnalyzerExecutor analyzerExecutor)
         {
-            Func<AnalyzerAndOptions, Task<HostCompilationStartAnalysisScope>> getTask = a =>
+            Func<Compilation, Task<HostCompilationStartAnalysisScope>> getTask = comp =>
             {
                 return Task.Run(() =>
                 {
@@ -65,8 +63,9 @@ internal partial class AnalyzerManager
                 }, analyzerExecutor.CancellationToken);
             };
 
-            var compilationActionsMap = _compilationScopeMap.GetValue(analyzerExecutor.Compilation, _compilationScopeMapCallback);
-            return compilationActionsMap.GetOrAdd(analyzerAndOptions, getTask);
+            var callback = new ConditionalWeakTable<Compilation, Task<HostCompilationStartAnalysisScope>>.CreateValueCallback(getTask);
+            var compilationActionsMap = _compilationScopeMap.GetOrAdd(analyzerAndOptions, new ConditionalWeakTable<Compilation, Task<HostCompilationStartAnalysisScope>>());
+            return compilationActionsMap.GetValue(analyzerExecutor.Compilation, callback);
         }
 
         private async Task<HostCompilationStartAnalysisScope> GetCompilationAnalysisScopeAsync(
@@ -84,9 +83,11 @@ internal partial class AnalyzerManager
             {
                 // Task to compute the scope was cancelled.
                 // Clear the entry in scope map for analyzer, so we can attempt a retry.
-                var compilationActionsMap = _compilationScopeMap.GetOrCreateValue(analyzerExecutor.Compilation);
-                Task<HostCompilationStartAnalysisScope> cancelledTask;
-                compilationActionsMap.TryRemove(analyzerAndOptions, out cancelledTask);
+                ConditionalWeakTable<Compilation, Task<HostCompilationStartAnalysisScope>> compilationActionsMap;
+                if (_compilationScopeMap.TryGetValue(analyzerAndOptions, out compilationActionsMap))
+                {
+                    compilationActionsMap.Remove(analyzerExecutor.Compilation);
+                }
 
                 analyzerExecutor.CancellationToken.ThrowIfCancellationRequested();
                 return await GetCompilationAnalysisScopeAsync(analyzer, sessionScope, analyzerExecutor).ConfigureAwait(false);
@@ -107,8 +108,7 @@ internal partial class AnalyzerManager
                 }, analyzerExecutor.CancellationToken);
             };
 
-            var callback = new ConditionalWeakTable<DiagnosticAnalyzer, Task<HostSessionStartAnalysisScope>>.CreateValueCallback(getTask);
-            return _sessionScopeMap.GetValue(analyzer, callback);
+            return _sessionScopeMap.GetOrAdd(analyzer, getTask);
         }
 
         private async Task<HostSessionStartAnalysisScope> GetSessionAnalysisScopeAsync(
@@ -123,7 +123,8 @@ internal partial class AnalyzerManager
             {
                 // Task to compute the scope was cancelled.
                 // Clear the entry in scope map for analyzer, so we can attempt a retry.
-                _sessionScopeMap.Remove(analyzer);
+                Task<HostSessionStartAnalysisScope> cancelledTask;
+                _sessionScopeMap.TryRemove(analyzer, out cancelledTask);
 
                 analyzerExecutor.CancellationToken.ThrowIfCancellationRequested();
                 return await GetSessionAnalysisScopeAsync(analyzer, analyzerExecutor).ConfigureAwait(false);
@@ -178,25 +179,30 @@ public async Task<bool> GetAnalyzerHasDependentCompilationEndAsync(DiagnosticAna
             DiagnosticAnalyzer analyzer,
             AnalyzerExecutor analyzerExecutor)
         {
-            var descriptors = _descriptorCache.GetValue(analyzer, key =>
+            var descriptors = _descriptorCache.GetOrAdd(analyzer, key =>
             {
                 var supportedDiagnostics = ImmutableArray<DiagnosticDescriptor>.Empty;
 
                 // Catch Exception from analyzer.SupportedDiagnostics
                 analyzerExecutor.ExecuteAndCatchIfThrows(analyzer, () => { supportedDiagnostics = analyzer.SupportedDiagnostics; });
 
-                var handler = new EventHandler<Exception>((sender, ex) =>
+                EventHandler<Exception> handler = null;
+                Action<Exception, DiagnosticAnalyzer, Diagnostic> onAnalyzerException = analyzerExecutor.OnAnalyzerException;
+                if (onAnalyzerException != null)
+                {
+                    handler = new EventHandler<Exception>((sender, ex) =>
                     {
                         var diagnostic = AnalyzerExecutor.GetAnalyzerExceptionDiagnostic(analyzer, ex);
-                        analyzerExecutor.OnAnalyzerException?.Invoke(ex, analyzer, diagnostic);
+                        onAnalyzerException(ex, analyzer, diagnostic);
                     });
 
-                // Subscribe for exceptions from lazily evaluated localizable strings in the descriptors.
-                foreach (var descriptor in supportedDiagnostics)
-                {
-                    descriptor.Title.OnException += handler;
-                    descriptor.MessageFormat.OnException += handler;
-                    descriptor.Description.OnException += handler;
+                    // Subscribe for exceptions from lazily evaluated localizable strings in the descriptors.
+                    foreach (var descriptor in supportedDiagnostics)
+                    {
+                        descriptor.Title.OnException += handler;
+                        descriptor.MessageFormat.OnException += handler;
+                        descriptor.Description.OnException += handler;
+                    }
                 }
 
                 return Tuple.Create(supportedDiagnostics, handler);
@@ -205,23 +211,54 @@ public async Task<bool> GetAnalyzerHasDependentCompilationEndAsync(DiagnosticAna
             return descriptors.Item1;
         }
 
-        internal void ClearAnalyzerExceptionHandlers(DiagnosticAnalyzer analyzer)
+        /// <summary>
+        /// This method should be invoked when the analyzer host is disposing off the analyzers.
+        /// It unregisters the exception handler hooked up to the descriptors' LocalizableString fields and subsequently removes the cached descriptors for the analyzers.
+        /// </summary>
+        internal void ClearAnalyzerState(ImmutableArray<DiagnosticAnalyzer> analyzers)
+        {
+            foreach (var analyzer in analyzers)
+            {
+                ClearDescriptorState(analyzer);
+                ClearAnalysisScopeState(analyzer);
+            }
+        }
+
+        private void ClearDescriptorState(DiagnosticAnalyzer analyzer)
         {
             // Host is disposing the analyzer instance, unsubscribe analyzer exception handlers.
             Tuple<ImmutableArray<DiagnosticDescriptor>, EventHandler<Exception>> value;
-            if (_descriptorCache.TryGetValue(analyzer, out value))
+            if (_descriptorCache.TryRemove(analyzer, out value))
             {
                 var descriptors = value.Item1;
                 var handler = value.Item2;
-                foreach (var descriptor in descriptors)
+                if (handler != null)
                 {
-                    descriptor.Title.OnException -= handler;
-                    descriptor.MessageFormat.OnException -= handler;
-                    descriptor.Description.OnException -= handler;
+                    foreach (var descriptor in descriptors)
+                    {
+                        descriptor.Title.OnException -= handler;
+                        descriptor.MessageFormat.OnException -= handler;
+                        descriptor.Description.OnException -= handler;
+                    }
                 }
             }
         }
 
+        private void ClearAnalysisScopeState(DiagnosticAnalyzer analyzer)
+        {
+            // Clear session scope.
+            Task<HostSessionStartAnalysisScope> canceledTask;
+            _sessionScopeMap.TryRemove(analyzer, out canceledTask);
+
+            // Clear compilation scope.
+            var keysToRemove = _compilationScopeMap.Keys.Where(analyzerAndOptions => analyzerAndOptions.Analyzer.Equals(analyzer)).ToImmutableArray();
+            foreach (var analyzerAndOptions in keysToRemove)
+            {
+                ConditionalWeakTable<Compilation, Task<HostCompilationStartAnalysisScope>> map;
+                _compilationScopeMap.TryRemove(analyzerAndOptions, out map);
+            }
+        }
+
         internal bool IsSupportedDiagnostic(DiagnosticAnalyzer analyzer, Diagnostic diagnostic, Func<DiagnosticAnalyzer, bool> isCompilerAnalyzer, AnalyzerExecutor analyzerExecutor)
         {
             // Avoid realizing all the descriptors for all compiler diagnostics by assuming that compiler analyzer doesn't report unsupported diagnostics.

src/Compilers/Core/Desktop/AnalyzerFileReference.AssemblyPathHelper.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.IO;
-using System.Reflection;
-using System.Reflection.Metadata;
-using System.Reflection.PortableExecutable;
-using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis.Diagnostics
 {

src/Compilers/Core/Desktop/AnalyzerFileReference.cs

@@ -85,13 +85,27 @@ public override ImmutableArray<DiagnosticAnalyzer> GetAnalyzersForAllLanguages()
         {
             if (_lazyAllAnalyzers.IsDefault)
             {
-                var allAnalyzers = MetadataCache.GetOrCreateAnalyzersFromFile(this);
-                ImmutableInterlocked.InterlockedInitialize(ref _lazyAllAnalyzers, allAnalyzers);
+                ImmutableInterlocked.InterlockedInitialize(ref _lazyAllAnalyzers, CreateAnalyzersForAllLanguages(this));
             }
 
             return _lazyAllAnalyzers;
         }
 
+        private static ImmutableArray<DiagnosticAnalyzer> CreateAnalyzersForAllLanguages(AnalyzerFileReference reference)
+        {
+            // Get all analyzers in the assembly.
+            var map = ImmutableDictionary.CreateBuilder<string, ImmutableArray<DiagnosticAnalyzer>>();
+            reference.AddAnalyzers(map);
+
+            var builder = ImmutableArray.CreateBuilder<DiagnosticAnalyzer>();
+            foreach (var analyzers in map.Values)
+            {
+                builder.AddRange(analyzers);
+            }
+
+            return builder.ToImmutable();
+        }
+
         public override ImmutableArray<DiagnosticAnalyzer> GetAnalyzers(string language)
         {
             if (string.IsNullOrEmpty(language))
@@ -102,9 +116,12 @@ public override ImmutableArray<DiagnosticAnalyzer> GetAnalyzers(string language)
             return ImmutableInterlocked.GetOrAdd(ref _lazyAnalyzersPerLanguage, language, CreateLanguageSpecificAnalyzers, this);
         }
 
-        private static ImmutableArray<DiagnosticAnalyzer> CreateLanguageSpecificAnalyzers(string language, AnalyzerFileReference @this)
+        private static ImmutableArray<DiagnosticAnalyzer> CreateLanguageSpecificAnalyzers(string langauge, AnalyzerFileReference reference)
         {
-            return MetadataCache.GetOrCreateAnalyzersFromFile(@this, language);
+            // Get all analyzers in the assembly for the given language.
+            var builder = ImmutableArray.CreateBuilder<DiagnosticAnalyzer>();
+            reference.AddAnalyzers(builder, langauge);
+            return builder.ToImmutable();
         }
 
         public override string FullPath

src/Compilers/Core/Desktop/CommandLine/CommonCompiler.cs

@@ -360,13 +360,14 @@ private int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, Cancellat
             cancellationToken.ThrowIfCancellationRequested();
 
             CancellationTokenSource analyzerCts = null;
+            AnalyzerManager analyzerManager = null;
             try
             {
                 Func<ImmutableArray<Diagnostic>> getAnalyzerDiagnostics = null;
                 if (!analyzers.IsDefaultOrEmpty)
                 {
                     analyzerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-                    var analyzerManager = new AnalyzerManager();
+                    analyzerManager = new AnalyzerManager();
                     var analyzerExceptionDiagnostics = new ConcurrentSet<Diagnostic>();
                     Action<Diagnostic> addExceptionDiagnostic = diagnostic => analyzerExceptionDiagnostics.Add(diagnostic);
                     var analyzerOptions = new AnalyzerOptions(ImmutableArray.Create<AdditionalText, AdditionalTextFile>(additionalTextFiles));
@@ -527,6 +528,9 @@ private int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, Cancellat
                 if (analyzerCts != null)
                 {
                     analyzerCts.Cancel();
+
+                    // Clear cached analyzer descriptors and unregister exception handlers hooked up to the LocalizableString fields of the associated descriptors.
+                    analyzerManager.ClearAnalyzerState(analyzers);
                 }
             }
 

src/Compilers/Core/Desktop/MetadataCache.cs

@@ -2,12 +2,9 @@
 
 using System;
 using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Threading;
-using Microsoft.CodeAnalysis.Diagnostics;
 using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis
@@ -20,8 +17,6 @@ namespace Microsoft.CodeAnalysis
     ///     2) A list of weak references to instances of VB/CS AssemblySymbols based on the PEAssembly object.
     ///
     /// For modules - a map from file name and timestamp to a weak reference to the corresponding PEModule object
-    /// 
-    /// For analyzer assemblies - a map from file name and timestamp to a weak reference to the diagnostic analyzers defined in the assembly.
     /// </summary>
     [Obsolete("To be removed", error: false)]
     internal static class MetadataCache
@@ -42,14 +37,6 @@ internal static class MetadataCache
 
         private static List<FileKey> s_moduleKeys = new List<FileKey>();
 
-        /// <summary>
-        /// Global cache for diagnostic analyzers imported from analyzer assembly files.
-        /// </summary>
-        private static Dictionary<FileKey, CachedAnalyzers> s_analyzersFromFiles =
-            new Dictionary<FileKey, CachedAnalyzers>();
-
-        private static List<FileKey> s_analyzerAssemblyKeys = new List<FileKey>();
-
         /// <summary>
         /// Timer triggering compact operation for metadata cache.
         /// </summary>
@@ -113,22 +100,6 @@ public CachedModule(ModuleMetadata metadata)
             }
         }
 
-        internal struct CachedAnalyzers
-        {
-            // Save a reference to the cached analyzers so that they don't get collected 
-            // if the metadata object gets collected.
-            public readonly WeakReference Analyzers;
-            public readonly string Language;
-
-            public CachedAnalyzers(object analyzers, string language)
-            {
-                Debug.Assert(analyzers != null);
-
-                this.Analyzers = new WeakReference(analyzers);
-                this.Language = language;
-            }
-        }
-
         /// <summary>
         /// Lock that must be acquired for the duration of read/write operations on MetadataCache.
         /// 
@@ -182,13 +153,12 @@ private static void CompactCache(Object state)
 
                 CompactCacheOfAssemblies();
                 CompactCacheOfModules();
-                CompactCacheOfAnalyzers();
 
                 s_compactCollectionCount = currentCollectionCount;
 
                 lock (Guard)
                 {
-                    if (!(AnyAssembliesCached() || AnyModulesCached() || AnyAnalyzerAssembliesCached()))
+                    if (!(AnyAssembliesCached() || AnyModulesCached()))
                     {
                         // Stop the timer
                         s_compactTimerIsOn = no;
@@ -392,71 +362,6 @@ private static bool AnyModulesCached()
             return s_moduleKeys.Count > 0;
         }
 
-        /// <summary>
-        /// Called by compactTimer.
-        /// </summary>
-        private static void CompactCacheOfAnalyzers()
-        {
-            // Do one pass through the analyzerAssemblyKeys list    
-            int originalCount = -1;
-
-            for (int current = 0; ; current++)
-            {
-                // Compact analyzer assemblies, one assembly per lock
-
-                // Lock our cache
-                lock (Guard)
-                {
-                    if (originalCount == -1)
-                    {
-                        originalCount = s_analyzerAssemblyKeys.Count;
-                    }
-
-                    if (s_analyzerAssemblyKeys.Count > current)
-                    {
-                        CachedAnalyzers cahedAnalyzers;
-                        FileKey key = s_analyzerAssemblyKeys[current];
-
-                        if (s_analyzersFromFiles.TryGetValue(key, out cahedAnalyzers))
-                        {
-                            if (!cahedAnalyzers.Analyzers.IsAlive)
-                            {
-                                // Analyzers has been collected
-                                s_analyzersFromFiles.Remove(key);
-                                s_analyzerAssemblyKeys.RemoveAt(current);
-                                current--;
-                            }
-                        }
-                        else
-                        {
-                            // Key is not found. Shouldn't ever get here!
-                            System.Diagnostics.Debug.Assert(false);
-                            s_analyzerAssemblyKeys.RemoveAt(current);
-                            current--;
-                        }
-                    }
-
-                    if (s_analyzerAssemblyKeys.Count <= current + 1)
-                    {
-                        // no more assemblies to process
-                        if (originalCount > s_analyzerAssemblyKeys.Count)
-                        {
-                            s_analyzerAssemblyKeys.TrimExcess();
-                        }
-
-                        return;
-                    }
-                }
-
-                Thread.Yield();
-            }
-        }
-
-        private static bool AnyAnalyzerAssembliesCached()
-        {
-            return s_analyzerAssemblyKeys.Count > 0;
-        }
-
         /// <summary>
         /// Global cache for assemblies imported from files.
         /// Internal accessibility is for test purpose only.
@@ -504,29 +409,6 @@ internal static List<FileKey> ModuleKeys
             }
         }
 
-        /// <summary>
-        /// Global cache for analyzers imported from files.
-        /// Internal accessibility is for test purpose only.
-        /// </summary>
-        internal static Dictionary<FileKey, CachedAnalyzers> AnalyzersFromFiles
-        {
-            get
-            {
-                return s_analyzersFromFiles;
-            }
-        }
-
-        /// <summary>
-        /// For test purposes only.
-        /// </summary>
-        internal static List<FileKey> AnalyzerAssemblyKeys
-        {
-            get
-            {
-                return s_analyzerAssemblyKeys;
-            }
-        }
-
         // for testing
         internal static CleaningCacheLock LockAndClean()
         {
@@ -545,8 +427,6 @@ internal class CleaningCacheLock : IDisposable
             private List<FileKey> _saveAssemblyKeys;
             private Dictionary<FileKey, CachedModule> _saveModulesFromFiles;
             private List<FileKey> _saveModuleKeys;
-            private Dictionary<FileKey, CachedAnalyzers> _saveAnalyzersFromFiles;
-            private List<FileKey> _saveAnalyzerAssemblyKeys;
 
             private bool _cacheIsLocked;
 
@@ -571,22 +451,16 @@ public static CleaningCacheLock LockAndCleanCaches()
                     result._saveAssemblyKeys = s_assemblyKeys;
                     result._saveModulesFromFiles = s_modulesFromFiles;
                     result._saveModuleKeys = s_moduleKeys;
-                    result._saveAnalyzersFromFiles = s_analyzersFromFiles;
-                    result._saveAnalyzerAssemblyKeys = s_analyzerAssemblyKeys;
 
                     var newAssembliesFromFiles = new Dictionary<FileKey, CachedAssembly>();
                     var newAssemblyKeys = new List<FileKey>();
                     var newModulesFromFiles = new Dictionary<FileKey, CachedModule>();
                     var newModuleKeys = new List<FileKey>();
-                    var newAnalyzersFromFiles = new Dictionary<FileKey, CachedAnalyzers>();
-                    var newAnalyzerAssemblyKeys = new List<FileKey>();
 
                     s_assembliesFromFiles = newAssembliesFromFiles;
                     s_assemblyKeys = newAssemblyKeys;
                     s_modulesFromFiles = newModulesFromFiles;
                     s_moduleKeys = newModuleKeys;
-                    s_analyzersFromFiles = newAnalyzersFromFiles;
-                    s_analyzerAssemblyKeys = newAnalyzerAssemblyKeys;
 
                     result._threadId = Thread.CurrentThread.ManagedThreadId;
                     result._stackTrace = Environment.StackTrace;
@@ -632,8 +506,6 @@ public void CleanCaches()
                 s_assemblyKeys.Clear();
                 s_modulesFromFiles.Clear();
                 s_moduleKeys.Clear();
-                s_analyzersFromFiles.Clear();
-                s_analyzerAssemblyKeys.Clear();
             }
 
             public void Dispose()
@@ -646,8 +518,6 @@ public void Dispose()
                     s_assemblyKeys = _saveAssemblyKeys;
                     s_modulesFromFiles = _saveModulesFromFiles;
                     s_moduleKeys = _saveModuleKeys;
-                    s_analyzersFromFiles = _saveAnalyzersFromFiles;
-                    s_analyzerAssemblyKeys = _saveAnalyzerAssemblyKeys;
 
                     Debug.Assert(ReferenceEquals(s_last, this));
                     Debug.Assert(_threadId == Thread.CurrentThread.ManagedThreadId);
@@ -698,97 +568,6 @@ internal static Metadata GetOrCreateFromFile(string fullPath, MetadataImageKind
             }
         }
 
-        internal static ImmutableArray<DiagnosticAnalyzer> GetOrCreateAnalyzersFromFile(AnalyzerFileReference analyzerReference, string langauge = null)
-        {
-            string fullPath = analyzerReference.FullPath;
-            Debug.Assert(PathUtilities.IsAbsolute(fullPath));
-
-            lock (Guard)
-            {
-                // may throw:
-                FileKey key = FileKey.Create(fullPath);
-
-                CachedAnalyzers cachedAnalyzers;
-                if (s_analyzersFromFiles.TryGetValue(key, out cachedAnalyzers))
-                {
-                    if (cachedAnalyzers.Analyzers.IsAlive && cachedAnalyzers.Language == langauge)
-                    {
-                        return (ImmutableArray<DiagnosticAnalyzer>)cachedAnalyzers.Analyzers.Target;
-                    }
-                    else
-                    {
-                        s_analyzersFromFiles.Remove(key);
-                        var removed = s_analyzerAssemblyKeys.Remove(key);
-                        Debug.Assert(removed);
-                        Debug.Assert(!s_analyzerAssemblyKeys.Contains(key));
-                    }
-                }
-
-                if (langauge == null)
-                {
-                    return CreateAnalyzersFromFile(analyzerReference);
-                }
-
-                return CreateAnalyzersFromFile(analyzerReference, langauge);
-            }
-        }
-
-        private static ImmutableArray<DiagnosticAnalyzer> CreateAnalyzersFromFile(AnalyzerFileReference reference)
-        {
-            Debug.Assert(PathUtilities.IsAbsolute(reference.FullPath));
-
-            // get all analyzers in the assembly;
-            var map = ImmutableDictionary.CreateBuilder<string, ImmutableArray<DiagnosticAnalyzer>>();
-            reference.AddAnalyzers(map);
-
-            // TODO: fix the cache mechanism. I don't understand how the cache is supposed to work
-            // so I am leaving it as it is for now. (does weak reference things currently actually work?)
-            // also, current one looks like assume a file can have analzyers for only one language.
-            // is this assumption right?
-            //
-            // foreach (var kv in mapBuilder)
-            // {
-            //     CacheAnalyzers(kv.Key, fullPath, kv.Value);
-            // }
-            //
-            // EnableCompactTimer();
-
-            var array = ImmutableArray.CreateBuilder<DiagnosticAnalyzer>();
-            foreach (var analyzers in map.Values)
-            {
-                array.AddRange(analyzers);
-            }
-
-            return array.ToImmutable();
-        }
-
-        private static ImmutableArray<DiagnosticAnalyzer> CreateAnalyzersFromFile(AnalyzerFileReference reference, string langauge)
-        {
-            Debug.Assert(PathUtilities.IsAbsolute(reference.FullPath));
-
-            // get all analyzers in the assembly for the given language;
-            var builder = ImmutableArray.CreateBuilder<DiagnosticAnalyzer>();
-            reference.AddAnalyzers(builder, langauge);
-            var analyzers = builder.ToImmutable();
-
-            CacheAnalyzers(langauge, reference.FullPath, analyzers);
-            EnableCompactTimer();
-
-            return analyzers;
-        }
-
-        private static void CacheAnalyzers(string langauge, string fullPath, ImmutableArray<DiagnosticAnalyzer> analyzers)
-        {
-            // refresh the timestamp (the file may have changed just before we memory-mapped it):
-            var key = FileKey.Create(fullPath);
-
-            s_analyzersFromFiles[key] = new CachedAnalyzers(analyzers, langauge);
-            Debug.Assert(!s_analyzerAssemblyKeys.Contains(key));
-
-            s_analyzerAssemblyKeys.Add(key);
-        }
-
-
         /// <exception cref="IOException"/>
         private static AssemblyMetadata GetOrCreateAssemblyFromFile(string fullPath)
         {

src/Compilers/Core/Portable/DiagnosticAnalyzer/AnalyzerDriver.cs

@@ -948,10 +948,24 @@ yield return
 
             var nodesToAnalyze = descendantDeclsToSkip == null ?
                 declaredNode.DescendantNodesAndSelf(descendIntoTrivia: true) :
-                declaredNode.DescendantNodesAndSelf(n => !descendantDeclsToSkip.Contains(n), descendIntoTrivia: true).Except(descendantDeclsToSkip);
+                GetSyntaxNodesToAnalyze(declaredNode, descendantDeclsToSkip);
 
             analyzerExecutor.ExecuteSyntaxNodeActions(nodesToAnalyze, actionsByKind, semanticModel, getKind);
         }
+
+        private static IEnumerable<SyntaxNode> GetSyntaxNodesToAnalyze(SyntaxNode declaredNode, HashSet<SyntaxNode> descendantDeclsToSkip)
+        {
+            Debug.Assert(declaredNode != null);
+            Debug.Assert(descendantDeclsToSkip != null);
+
+            foreach (var node in declaredNode.DescendantNodesAndSelf(n => !descendantDeclsToSkip.Contains(n), descendIntoTrivia: true))
+            {
+                if (!descendantDeclsToSkip.Contains(node))
+                {
+                    yield return node;
+                }
+            }
+        }
     }
 
     internal static class AnalyzerDriverResources

src/EditorFeatures/Test/Diagnostics/TestDiagnosticAnalyzerService.cs

@@ -41,14 +41,14 @@ internal TestDiagnosticAnalyzerService(ImmutableArray<AnalyzerReference> workspa
             _onAnalyzerException = onAnalyzerException;
         }
 
-        internal override Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException(Project project, DiagnosticLogAggregator diagnosticLogAggregator)
+        internal override Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException(ProjectId projectId, DiagnosticLogAggregator diagnosticLogAggregator)
         {
-            return _onAnalyzerException ?? base.GetOnAnalyzerException(project, diagnosticLogAggregator);
+            return _onAnalyzerException ?? base.GetOnAnalyzerException(projectId, diagnosticLogAggregator);
         }
 
-        internal override Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException_NoTelemetryLogging(Project project)
+        internal override Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException_NoTelemetryLogging(ProjectId projectId)
         {
-            return _onAnalyzerException ?? base.GetOnAnalyzerException_NoTelemetryLogging(project);
+            return _onAnalyzerException ?? base.GetOnAnalyzerException_NoTelemetryLogging(projectId);
         }
 
         private class TestAnalyzerReferenceByLanguage : AnalyzerReference

src/Features/Core/Diagnostics/AbstractHostDiagnosticUpdateSource.cs

@@ -44,13 +44,15 @@ protected void RaiseDiagnosticsUpdated(DiagnosticsUpdatedArgs args)
             }
         }
 
-        internal void ReportAnalyzerDiagnostic(DiagnosticAnalyzer analyzer, Diagnostic diagnostic, Workspace workspace, Project project)
+        internal void ReportAnalyzerDiagnostic(DiagnosticAnalyzer analyzer, Diagnostic diagnostic, Workspace workspace, ProjectId projectId)
         {
             if (workspace != this.Workspace)
             {
                 return;
             }
 
+            var project = workspace.CurrentSolution.GetProject(projectId);
+
             bool raiseDiagnosticsUpdated = true;
             var diagnosticData = project != null ?
                 DiagnosticData.Create(project, diagnostic) :
@@ -74,10 +76,16 @@ internal void ReportAnalyzerDiagnostic(DiagnosticAnalyzer analyzer, Diagnostic d
 
         public void ClearAnalyzerReferenceDiagnostics(AnalyzerFileReference analyzerReference, string language, ProjectId projectId)
         {
-            foreach (var analyzer in analyzerReference.GetAnalyzers(language))
+            var analyzers = analyzerReference.GetAnalyzers(language);
+            ClearAnalyzerDiagnostics(analyzers, projectId);
+            AnalyzerManager.Instance.ClearAnalyzerState(analyzers);
+        }
+
+        private void ClearAnalyzerDiagnostics(ImmutableArray<DiagnosticAnalyzer> analyzers, ProjectId projectId)
+        {
+            foreach (var analyzer in analyzers)
             {
                 ClearAnalyzerDiagnostics(analyzer, projectId);
-                AnalyzerManager.Instance.ClearAnalyzerExceptionHandlers(analyzer);
             }
         }
 

src/Features/Core/Diagnostics/AnalyzerHelper.cs

@@ -62,7 +62,7 @@ public static bool IsCompilerAnalyzer(this DiagnosticAnalyzer analyzer)
             DiagnosticAnalyzer analyzer,
             Diagnostic diagnostic,
             AbstractHostDiagnosticUpdateSource hostDiagnosticUpdateSource,
-            Project projectOpt = null)
+            ProjectId projectOpt = null)
         {
             if (diagnostic != null)
             {

src/Features/Core/Diagnostics/BaseDiagnosticIncrementalAnalyzer.cs

@@ -187,15 +187,15 @@ public virtual void LogAnalyzerCountSummary()
         }
 
         // internal for testing purposes.
-        internal Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException(Project project)
+        internal Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException(ProjectId projectId)
         {
-            return Owner?.GetOnAnalyzerException(project, DiagnosticLogAggregator);
+            return Owner?.GetOnAnalyzerException(projectId, DiagnosticLogAggregator);
         }
 
         // internal for testing purposes.
-        internal Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException_NoTelemetryLogging(Project project)
+        internal Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException_NoTelemetryLogging(ProjectId projectId)
         {
-            return Owner?.GetOnAnalyzerException_NoTelemetryLogging(project);
+            return Owner?.GetOnAnalyzerException_NoTelemetryLogging(projectId);
         }
     }
 }

src/Features/Core/Diagnostics/DiagnosticAnalyzerService.cs

@@ -170,22 +170,22 @@ public Task<ImmutableArray<DiagnosticData>> GetSpecificDiagnosticsAsync(Solution
         }
 
         // virtual for testing purposes.
-        internal virtual Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException(Project project, DiagnosticLogAggregator diagnosticLogAggregator)
+        internal virtual Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException(ProjectId projectId, DiagnosticLogAggregator diagnosticLogAggregator)
         {
             return (ex, analyzer, diagnostic) =>
             {
                 // Log telemetry, if analyzer supports telemetry.
                 DiagnosticAnalyzerLogger.LogAnalyzerCrashCount(analyzer, ex, diagnosticLogAggregator);
 
-                AnalyzerHelper.OnAnalyzerException_NoTelemetryLogging(ex, analyzer, diagnostic, _hostDiagnosticUpdateSource, project);
+                AnalyzerHelper.OnAnalyzerException_NoTelemetryLogging(ex, analyzer, diagnostic, _hostDiagnosticUpdateSource, projectId);
             };
         }
 
         // virtual for testing purposes.
-        internal virtual Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException_NoTelemetryLogging(Project project)
+        internal virtual Action<Exception, DiagnosticAnalyzer, Diagnostic> GetOnAnalyzerException_NoTelemetryLogging(ProjectId projectId)
         {
             return (ex, analyzer, diagnostic) =>
-                AnalyzerHelper.OnAnalyzerException_NoTelemetryLogging(ex, analyzer, diagnostic, _hostDiagnosticUpdateSource, project);
+                AnalyzerHelper.OnAnalyzerException_NoTelemetryLogging(ex, analyzer, diagnostic, _hostDiagnosticUpdateSource, projectId);
         }
     }
 }

src/Features/Core/Diagnostics/EngineV1/DiagnosticAnalyzerDriver.cs

@@ -64,8 +64,8 @@ internal class DiagnosticAnalyzerDriver
             _generatedCodeService = project.Solution.Workspace.Services.GetService<IGeneratedCodeRecognitionService>();
             _analyzerDriverService = project.LanguageServices.GetService<IAnalyzerDriverService>();
             _analyzerOptions = new WorkspaceAnalyzerOptions(project.AnalyzerOptions, project.Solution.Workspace);
-            _onAnalyzerException = owner.GetOnAnalyzerException(project);
-            _onAnalyzerException_NoTelemetryLogging = owner.GetOnAnalyzerException_NoTelemetryLogging(project);
+            _onAnalyzerException = owner.GetOnAnalyzerException(project.Id);
+            _onAnalyzerException_NoTelemetryLogging = owner.GetOnAnalyzerException_NoTelemetryLogging(project.Id);
         }
 
         public Document Document