prevent re-analyze to create big hashset (#22693)

* log more info on reanalyze, and prevent re-analyze to create big hash set.

* removed unnecessary log and fixed test failure

* added reanalyze scope

* PR feedbacks.

src/Features/Core/Portable/SolutionCrawler/SolutionCrawlerLogger.cs

@@ -10,12 +10,13 @@
 
 namespace Microsoft.CodeAnalysis.SolutionCrawler
 {
-    internal class SolutionCrawlerLogger
+    internal static class SolutionCrawlerLogger
     {
         private const string Id = nameof(Id);
         private const string Kind = nameof(Kind);
         private const string Analyzer = nameof(Analyzer);
         private const string DocumentCount = nameof(DocumentCount);
+        private const string Languages = nameof(Languages);
         private const string HighPriority = nameof(HighPriority);
         private const string Enabled = nameof(Enabled);
         private const string AnalyzerCount = nameof(AnalyzerCount);
@@ -67,14 +68,20 @@ public static void LogUnregistration(int correlationId)
             }));
         }
 
-        public static void LogReanalyze(int correlationId, IIncrementalAnalyzer analyzer, IEnumerable<DocumentId> documentIds, bool highPriority)
+        public static void LogReanalyze(
+            int correlationId, 
+            IIncrementalAnalyzer analyzer, 
+            int documentCount,
+            string languages,
+            bool highPriority)
         {
             Logger.Log(FunctionId.WorkCoordinatorRegistrationService_Reanalyze, KeyValueLogMessage.Create(m =>
             {
                 m[Id] = correlationId;
                 m[Analyzer] = analyzer.ToString();
-                m[DocumentCount] = documentIds == null ? 0 : documentIds.Count();
+                m[DocumentCount] = documentCount;
                 m[HighPriority] = highPriority;
+                m[Languages] = languages;
             }));
         }
 

src/Features/Core/Portable/SolutionCrawler/SolutionCrawlerRegistrationService.cs

@@ -134,22 +134,11 @@ public void Reanalyze(Workspace workspace, IIncrementalAnalyzer analyzer, IEnume
                 // no specific projects or documents provided
                 if (projectIds == null && documentIds == null)
                 {
-                    coordinator.Reanalyze(analyzer, workspace.CurrentSolution.Projects.SelectMany(p => p.DocumentIds).ToSet(), highPriority);
+                    coordinator.Reanalyze(analyzer, new ReanalyzeScope(workspace.CurrentSolution.Id), highPriority);
                     return;
                 }
 
-                // specific documents provided
-                if (projectIds == null)
-                {
-                    coordinator.Reanalyze(analyzer, documentIds.ToSet(), highPriority);
-                    return;
-                }
-
-                var solution = workspace.CurrentSolution;
-                var set = new HashSet<DocumentId>(documentIds ?? SpecializedCollections.EmptyEnumerable<DocumentId>());
-                set.UnionWith(projectIds.Select(id => solution.GetProject(id)).SelectMany(p => p.DocumentIds));
-
-                coordinator.Reanalyze(analyzer, set, highPriority);
+                coordinator.Reanalyze(analyzer, new ReanalyzeScope(projectIds, documentIds), highPriority);
             }
         }
 

src/Features/Core/Portable/SolutionCrawler/WorkCoordinator.cs

@@ -87,8 +87,8 @@ public void AddAnalyzer(IIncrementalAnalyzer analyzer, bool highPriorityForActiv
                 _documentAndProjectWorkerProcessor.AddAnalyzer(analyzer, highPriorityForActiveFile);
 
                 // and ask to re-analyze whole solution for the given analyzer
-                var set = _registration.CurrentSolution.Projects.SelectMany(p => p.DocumentIds).ToSet();
-                Reanalyze(analyzer, set);
+                var scope = new ReanalyzeScope(_registration.CurrentSolution.Id);
+                Reanalyze(analyzer, scope);
             }
 
             public void Shutdown(bool blockingShutdown)
@@ -151,29 +151,30 @@ private void OnOptionChanged(object sender, OptionChangedEventArgs e)
 
             private void ReanalyzeOnOptionChange(object sender, OptionChangedEventArgs e)
             {
-                // otherwise, let each analyzer decide what they want on option change
-                ISet<DocumentId> set = null;
+                // let each analyzer decide what they want on option change
                 foreach (var analyzer in _documentAndProjectWorkerProcessor.Analyzers)
                 {
                     if (analyzer.NeedsReanalysisOnOptionChanged(sender, e))
                     {
-                        set = set ?? _registration.CurrentSolution.Projects.SelectMany(p => p.DocumentIds).ToSet();
-                        this.Reanalyze(analyzer, set);
+                        var scope = new ReanalyzeScope(_registration.CurrentSolution.Id);
+                        Reanalyze(analyzer, scope);
                     }
                 }
             }
 
-            public void Reanalyze(IIncrementalAnalyzer analyzer, ISet<DocumentId> documentIds, bool highPriority = false)
+            public void Reanalyze(IIncrementalAnalyzer analyzer, ReanalyzeScope scope, bool highPriority = false)
             {
                 var asyncToken = _listener.BeginAsyncOperation("Reanalyze");
                 _eventProcessingQueue.ScheduleTask(
-                    () => EnqueueWorkItemAsync(analyzer, documentIds, highPriority), _shutdownToken).CompletesAsyncOperation(asyncToken);
+                    () => EnqueueWorkItemAsync(analyzer, scope, highPriority), _shutdownToken).CompletesAsyncOperation(asyncToken);
 
-                if (documentIds?.Count > 1)
+                if (scope.HasMultipleDocuments)
                 {
                     // log big reanalysis request from things like fix all, suppress all or option changes
                     // we are not interested in 1 file re-analysis request which can happen from like venus typing
-                    SolutionCrawlerLogger.LogReanalyze(CorrelationId, analyzer, documentIds, highPriority);
+                    var solution = _registration.CurrentSolution;
+                    SolutionCrawlerLogger.LogReanalyze(
+                        CorrelationId, analyzer, scope.GetDocumentCount(solution), scope.GetLanguages(solution), highPriority);
                 }
             }
 
@@ -414,26 +415,25 @@ private async Task EnqueueWorkItemAsync(Project project, InvocationReasons invoc
                 }
             }
 
-            private async Task EnqueueWorkItemAsync(IIncrementalAnalyzer analyzer, IEnumerable<DocumentId> documentIds, bool highPriority)
+            private async Task EnqueueWorkItemAsync(IIncrementalAnalyzer analyzer, ReanalyzeScope scope, bool highPriority)
             {
                 var solution = _registration.CurrentSolution;
-                foreach (var documentId in documentIds)
-                {
-                    var document = solution.GetDocument(documentId);
-                    if (document == null)
-                    {
-                        continue;
-                    }
+                var invocationReasons = highPriority ? InvocationReasons.ReanalyzeHighPriority : InvocationReasons.Reanalyze;
 
-                    var priorityService = document.GetLanguageService<IWorkCoordinatorPriorityService>();
-                    var isLowPriority = priorityService != null && await priorityService.IsLowPriorityAsync(document, _shutdownToken).ConfigureAwait(false);
+                foreach (var document in scope.GetDocuments(solution))
+                {
+                    await EnqueueWorkItemAsync(analyzer, document, invocationReasons).ConfigureAwait(false);
+                }
+            }
 
-                    var invocationReasons = highPriority ? InvocationReasons.ReanalyzeHighPriority : InvocationReasons.Reanalyze;
+            private async Task EnqueueWorkItemAsync(IIncrementalAnalyzer analyzer, Document document, InvocationReasons invocationReasons)
+            {
+                var priorityService = document.GetLanguageService<IWorkCoordinatorPriorityService>();
+                var isLowPriority = priorityService != null && await priorityService.IsLowPriorityAsync(document, _shutdownToken).ConfigureAwait(false);
 
-                    _documentAndProjectWorkerProcessor.Enqueue(
-                        new WorkItem(documentId, document.Project.Language, invocationReasons,
-                        isLowPriority, analyzer, _listener.BeginAsyncOperation("WorkItem")));
-                }
+                _documentAndProjectWorkerProcessor.Enqueue(
+                    new WorkItem(document.Id, document.Project.Language, invocationReasons,
+                    isLowPriority, analyzer, _listener.BeginAsyncOperation("WorkItem")));
             }
 
             private async Task EnqueueWorkItemAsync(Solution oldSolution, Solution newSolution)
@@ -591,5 +591,157 @@ internal void WaitUntilCompletion_ForTestingPurposesOnly()
                 _documentAndProjectWorkerProcessor.WaitUntilCompletion_ForTestingPurposesOnly();
             }
         }
+
+        private struct ReanalyzeScope
+        {
+            private readonly SolutionId _solutionId;
+            private readonly ISet<object> _projectOrDocumentIds;
+
+            public ReanalyzeScope(SolutionId solutionId)
+            {
+                _solutionId = solutionId;
+                _projectOrDocumentIds = null;
+            }
+
+            public ReanalyzeScope(IEnumerable<ProjectId> projectIds = null, IEnumerable<DocumentId> documentIds = null)
+            {
+                projectIds = projectIds ?? SpecializedCollections.EmptyEnumerable<ProjectId>();
+                documentIds = documentIds ?? SpecializedCollections.EmptyEnumerable<DocumentId>();
+
+                _solutionId = null;
+                _projectOrDocumentIds = new HashSet<object>(projectIds);
+
+                foreach (var documentId in documentIds)
+                {
+                    if (_projectOrDocumentIds.Contains(documentId.ProjectId))
+                    {
+                        continue;
+                    }
+
+                    _projectOrDocumentIds.Add(documentId);
+                }
+            }
+
+            public bool HasMultipleDocuments => _solutionId != null || _projectOrDocumentIds?.Count > 1;
+
+            public string GetLanguages(Solution solution)
+            {
+                Contract.ThrowIfFalse(_solutionId == null || solution.Id == _solutionId);
+
+                using (var pool = SharedPools.Default<HashSet<string>>().GetPooledObject())
+                {
+                    if (_solutionId != null)
+                    {
+                        pool.Object.UnionWith(solution.State.ProjectStates.Select(kv => kv.Value.Language));
+                        return string.Join(",", pool.Object);
+                    }
+
+                    foreach (var projectOrDocumentId in _projectOrDocumentIds)
+                    {
+                        switch (projectOrDocumentId)
+                        {
+                            case ProjectId projectId:
+                                var project = solution.GetProject(projectId);
+                                if (project != null)
+                                {
+                                    pool.Object.Add(project.Language);
+                                }
+                                break;
+                            case DocumentId documentId:
+                                var document = solution.GetDocument(documentId);
+                                if (document != null)
+                                {
+                                    pool.Object.Add(document.Project.Language);
+                                }
+                                break;
+                            default:
+                                throw ExceptionUtilities.UnexpectedValue(projectOrDocumentId);
+                        }
+                    }
+
+                    return string.Join(",", pool.Object);
+                }
+            }
+
+            public int GetDocumentCount(Solution solution)
+            {
+                Contract.ThrowIfFalse(_solutionId == null || solution.Id == _solutionId);
+
+                var count = 0;
+                if (_solutionId != null)
+                {
+                    foreach (var projectState in solution.State.ProjectStates)
+                    {
+                        count += projectState.Value.DocumentIds.Count;
+                    }
+
+                    return count;
+                }
+
+                foreach (var projectOrDocumentId in _projectOrDocumentIds)
+                {
+                    switch (projectOrDocumentId)
+                    {
+                        case ProjectId projectId:
+                            var project = solution.GetProject(projectId);
+                            if (project != null)
+                            {
+                                count += project.DocumentIds.Count;
+                            }
+                            break;
+                        case DocumentId documentId:
+                            count++;
+                            break;
+                        default:
+                            throw ExceptionUtilities.UnexpectedValue(projectOrDocumentId);
+                    }
+                }
+
+                return count;
+            }
+
+            public IEnumerable<Document> GetDocuments(Solution solution)
+            {
+                Contract.ThrowIfFalse(_solutionId == null || solution.Id == _solutionId);
+
+                if (_solutionId != null)
+                {
+                    foreach (var document in solution.Projects.SelectMany(p => p.Documents))
+                    {
+                        yield return document;
+                    }
+
+                    yield break;
+                }
+
+                foreach (var projectOrDocumentId in _projectOrDocumentIds)
+                {
+                    switch (projectOrDocumentId)
+                    {
+                        case ProjectId projectId:
+                        {
+                            var project = solution.GetProject(projectId);
+                            if (project != null)
+                            {
+                                foreach (var document in project.Documents)
+                                {
+                                    yield return document;
+                                }
+                            }
+                            break;
+                        }
+                        case DocumentId documentId:
+                        {
+                            var document = solution.GetDocument(documentId);
+                            if (document != null)
+                            {
+                                yield return document;
+                            }
+                            break;
+                        }
+                    }
+                }
+            }
+        }
     }
 }