Merge pull request #20271 from CyrusNajmabadi/internIndexStrings

Intern the strings we create for our SyntaxTree index.

src/EditorFeatures/TestUtilities/TestExportProvider.cs

@@ -56,6 +56,8 @@ private static Type[] GetNeutralAndCSharpAndVisualBasicTypes()
                 typeof(CodeAnalysis.VisualBasic.CodeGeneration.VisualBasicCodeGenerationServiceFactory),
                 typeof(CodeAnalysis.CSharp.CSharpSyntaxFactsServiceFactory),
                 typeof(CodeAnalysis.VisualBasic.VisualBasicSyntaxFactsServiceFactory),
+                typeof(CodeAnalysis.CSharp.FindSymbols.CSharpDeclaredSymbolInfoFactoryService),
+                typeof(CodeAnalysis.VisualBasic.FindSymbols.VisualBasicDeclaredSymbolInfoFactoryService),
                 typeof(CodeAnalysis.CSharp.CSharpSymbolDeclarationService),
                 typeof(CodeAnalysis.VisualBasic.VisualBasicSymbolDeclarationService),
                 typeof(CSharp.LanguageServices.CSharpSymbolDisplayServiceFactory),

src/Workspaces/CSharp/Portable/CSharpWorkspace.csproj

@@ -98,6 +98,7 @@
     <Compile Include="CodeStyle\TypeStyle\TypeStyleHelper.cs" />
     <Compile Include="Composition\CSharpWorkspaceFeatures.cs" />
     <Compile Include="Extensions\BaseMethodDeclarationSyntaxExtensions.cs" />
+    <Compile Include="FindSymbols\CSharpDeclaredSymbolInfoFactoryService.cs" />
     <Compile Include="Simplification\Reducers\CSharpDefaultExpressionReducer.Rewriter.cs" />
     <Compile Include="Diagnostics\CSharpDiagnosticPropertiesService.cs" />
     <Compile Include="Execution\CSharpOptionsSerializationService.cs" />

src/Workspaces/Core/Portable/FindSymbols/DeclaredSymbolInfo.cs

@@ -2,7 +2,6 @@
 
 using System;
 using System.Collections.Immutable;
-using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.ErrorReporting;
@@ -82,6 +81,7 @@ internal struct DeclaredSymbolInfo
         public ImmutableArray<string> InheritanceNames { get; }
 
         public DeclaredSymbolInfo(
+            StringTable stringTable,
             string name,
             string nameSuffix,
             string containerDisplayName,
@@ -91,12 +91,11 @@ internal struct DeclaredSymbolInfo
             TextSpan span,
             ImmutableArray<string> inheritanceNames,
             int parameterCount = 0, int typeParameterCount = 0)
-            : this()
         {
-            Name = name;
-            NameSuffix = nameSuffix;
-            ContainerDisplayName = containerDisplayName;
-            FullyQualifiedContainerName = fullyQualifiedContainerName;
+            Name = Intern(stringTable, name);
+            NameSuffix = Intern(stringTable, nameSuffix);
+            ContainerDisplayName = Intern(stringTable, containerDisplayName);
+            FullyQualifiedContainerName = Intern(stringTable, fullyQualifiedContainerName);
             Span = span;
             InheritanceNames = inheritanceNames;
 
@@ -109,6 +108,9 @@ internal struct DeclaredSymbolInfo
             _flags = (uint)kind | ((uint)accessibility << 4) | ((uint)parameterCount << 8) | ((uint)typeParameterCount << 12);
         }
 
+        public static string Intern(StringTable stringTable, string name)
+            => name == null ? null : stringTable.Add(name);
+
         private static DeclaredSymbolInfoKind GetKind(uint flags)
             => (DeclaredSymbolInfoKind)(flags & Lower4BitMask);
 
@@ -138,7 +140,7 @@ internal void WriteTo(ObjectWriter writer)
             }
         }
 
-        internal static DeclaredSymbolInfo ReadFrom_ThrowsOnFailure(ObjectReader reader)
+        internal static DeclaredSymbolInfo ReadFrom_ThrowsOnFailure(StringTable stringTable, ObjectReader reader)
         {
             var name = reader.ReadString();
             var nameSuffix = reader.ReadString();
@@ -157,6 +159,7 @@ internal static DeclaredSymbolInfo ReadFrom_ThrowsOnFailure(ObjectReader reader)
 
             var span = new TextSpan(spanStart, spanLength);
             return new DeclaredSymbolInfo(
+                stringTable,
                 name: name,
                 nameSuffix: nameSuffix,
                 containerDisplayName: containerDisplayName,

src/Workspaces/Core/Portable/FindSymbols/SyntaxTree/SyntaxTreeIndex.DeclarationInfo.cs

@@ -26,7 +26,7 @@ public void WriteTo(ObjectWriter writer)
                 }
             }
 
-            public static DeclarationInfo? TryReadFrom(ObjectReader reader)
+            public static DeclarationInfo? TryReadFrom(StringTable stringTable, ObjectReader reader)
             {
                 try
                 {
@@ -34,7 +34,7 @@ public void WriteTo(ObjectWriter writer)
                     var builder = ImmutableArray.CreateBuilder<DeclaredSymbolInfo>(declaredSymbolCount);
                     for (int i = 0; i < declaredSymbolCount; i++)
                     {
-                        builder.Add(DeclaredSymbolInfo.ReadFrom_ThrowsOnFailure(reader));
+                        builder.Add(DeclaredSymbolInfo.ReadFrom_ThrowsOnFailure(stringTable, reader));
                     }
 
                     return new DeclarationInfo(builder.MoveToImmutable());

src/Workspaces/Core/Portable/FindSymbols/SyntaxTree/SyntaxTreeIndex_Create.cs

@@ -2,9 +2,11 @@
 
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.ErrorReporting;
+using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.LanguageServices;
 using Microsoft.CodeAnalysis.PooledObjects;
 using Microsoft.CodeAnalysis.Shared.Extensions;
@@ -13,6 +15,11 @@
 
 namespace Microsoft.CodeAnalysis.FindSymbols
 {
+    internal interface IDeclaredSymbolInfoFactoryService : ILanguageService
+    {
+        bool TryGetDeclaredSymbolInfo(StringTable stringTable, SyntaxNode node, out DeclaredSymbolInfo declaredSymbolInfo);
+    }
+
     internal sealed partial class SyntaxTreeIndex
     {
         // The probability of getting a false positive when calling ContainsIdentifier.
@@ -24,10 +31,26 @@ internal sealed partial class SyntaxTreeIndex
         public static readonly ObjectPool<HashSet<long>> LongLiteralHashSetPool =
             new ObjectPool<HashSet<long>>(() => new HashSet<long>(), 20);
 
+        /// <summary>
+        /// String interning table so that we can share many more strings in our DeclaredSymbolInfo
+        /// buckets.  Keyed off a Project instance so that we share all these strings as we create
+        /// the or load the index items for this a specific Project.  This helps as we will generally 
+        /// be creating or loading all the index items for the documents in a Project at the same time.
+        /// Once this project is let go of (which happens with any solution change) then we'll dump
+        /// this string table.  The table will have already served its purpose at that point and 
+        /// doesn't need to be kept around further.
+        /// </summary>
+        private static readonly ConditionalWeakTable<Project, StringTable> s_projectStringTable =
+            new ConditionalWeakTable<Project, StringTable>();
+
         private static async Task<SyntaxTreeIndex> CreateIndexAsync(
             Document document, Checksum checksum, CancellationToken cancellationToken)
         {
+            var project = document.Project;
+            var stringTable = GetStringTable(project);
+
             var syntaxFacts = document.GetLanguageService<ISyntaxFactsService>();
+            var infoFactory = document.GetLanguageService<IDeclaredSymbolInfoFactoryService>();
             var ignoreCase = syntaxFacts != null && !syntaxFacts.IsCaseSensitive;
             var isCaseSensitive = !ignoreCase;
 
@@ -68,6 +91,7 @@ internal sealed partial class SyntaxTreeIndex
                             containsQueryExpression = containsQueryExpression || syntaxFacts.IsQueryExpression(node);
                             containsElementAccess = containsElementAccess || syntaxFacts.IsElementAccessExpression(node);
                             containsIndexerMemberCref = containsIndexerMemberCref || syntaxFacts.IsIndexerMemberCRef(node);
+
                             // We've received a number of error reports where DeclaredSymbolInfo.GetSymbolAsync() will
                             // crash because the document's syntax root doesn't contain the span of the node returned
                             // by TryGetDeclaredSymbolInfo().  There are two possibilities for this crash:
@@ -78,7 +102,7 @@ internal sealed partial class SyntaxTreeIndex
                             // the future then we know the problem lies in (2).  If, however, the problem is really in
                             // TryGetDeclaredSymbolInfo, then this will at least prevent us from returning bad spans
                             // and will prevent the crash from occurring.
-                            if (syntaxFacts.TryGetDeclaredSymbolInfo(node, out var declaredSymbolInfo))
+                            if (infoFactory.TryGetDeclaredSymbolInfo(stringTable, node, out var declaredSymbolInfo))
                             {
                                 if (root.FullSpan.Contains(declaredSymbolInfo.Span))
                                 {
@@ -186,6 +210,9 @@ internal sealed partial class SyntaxTreeIndex
             }
         }
 
+        private static StringTable GetStringTable(Project project) 
+            => s_projectStringTable.GetValue(project, _ => StringTable.GetInstance());
+
         private static void GetIdentifierSet(bool ignoreCase, out HashSet<string> identifiers, out HashSet<string> escapedIdentifiers)
         {
             if (ignoreCase)

src/Workspaces/Core/Portable/FindSymbols/SyntaxTree/SyntaxTreeIndex_Persistence.cs

@@ -53,7 +53,7 @@ private void WriteFormatAndChecksum(ObjectWriter writer, string formatVersion)
                     {
                         if (FormatAndChecksumMatches(reader, SerializationFormat, checksum))
                         {
-                            return ReadFrom(reader, checksum);
+                            return ReadFrom(GetStringTable(document.Project), reader, checksum);
                         }
                     }
                 }
@@ -155,12 +155,12 @@ public void WriteTo(ObjectWriter writer)
         }
 
         private static SyntaxTreeIndex ReadFrom(
-            ObjectReader reader, Checksum checksum)
+            StringTable stringTable, ObjectReader reader, Checksum checksum)
         {
             var literalInfo = LiteralInfo.TryReadFrom(reader);
             var identifierInfo = IdentifierInfo.TryReadFrom(reader);
             var contextInfo = ContextInfo.TryReadFrom(reader);
-            var declarationInfo = DeclarationInfo.TryReadFrom(reader);
+            var declarationInfo = DeclarationInfo.TryReadFrom(stringTable, reader);
 
             if (literalInfo == null || identifierInfo == null || contextInfo == null || declarationInfo == null)
             {

src/Workspaces/Core/Portable/LanguageServices/SyntaxFactsService/AbstractSyntaxFactsService.cs

@@ -2,20 +2,21 @@
 
 using System;
 using System.Collections.Generic;
-using Microsoft.CodeAnalysis.PooledObjects;
-using Microsoft.CodeAnalysis.Shared.Extensions;
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Linq;
+using System.Text;
 using System.Threading;
+using Microsoft.CodeAnalysis.FindSymbols;
+using Microsoft.CodeAnalysis.PooledObjects;
+using Microsoft.CodeAnalysis.Shared.Extensions;
 using Microsoft.CodeAnalysis.Shared.Utilities;
 using Microsoft.CodeAnalysis.Text;
 using Roslyn.Utilities;
-using System.Text;
 
 namespace Microsoft.CodeAnalysis.LanguageServices
 {
-    internal abstract class AbstractSyntaxFactsService
+    internal abstract class AbstractDeclaredSymbolInfoFactoryService : IDeclaredSymbolInfoFactoryService
     {
         private readonly static ObjectPool<List<Dictionary<string, string>>> s_aliasMapListPool =
             new ObjectPool<List<Dictionary<string, string>>>(() => new List<Dictionary<string, string>>());
@@ -28,6 +29,61 @@ internal abstract class AbstractSyntaxFactsService
         private readonly static ObjectPool<Dictionary<string, string>> s_aliasMapPool =
             new ObjectPool<Dictionary<string, string>>(() => new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
 
+        protected static List<Dictionary<string, string>> AllocateAliasMapList()
+            => s_aliasMapListPool.Allocate();
+
+        protected static void FreeAliasMapList(List<Dictionary<string, string>> list)
+        {
+            if (list != null)
+            {
+                foreach (var aliasMap in list)
+                {
+                    FreeAliasMap(aliasMap);
+                }
+
+                s_aliasMapListPool.ClearAndFree(list);
+            }
+        }
+
+        protected static void FreeAliasMap(Dictionary<string, string> aliasMap)
+        {
+            if (aliasMap != null)
+            {
+                s_aliasMapPool.ClearAndFree(aliasMap);
+            }
+        }
+
+        protected static Dictionary<string, string> AllocateAliasMap()
+            => s_aliasMapPool.Allocate();
+
+        protected static void AppendTokens(SyntaxNode node, StringBuilder builder)
+        {
+            foreach (var child in node.ChildNodesAndTokens())
+            {
+                if (child.IsToken)
+                {
+                    builder.Append(child.AsToken().Text);
+                }
+                else
+                {
+                    AppendTokens(child.AsNode(), builder);
+                }
+            }
+        }
+
+        protected static void Intern(StringTable stringTable, ArrayBuilder<string> builder)
+        {
+            for (int i = 0, n = builder.Count; i < n; i++)
+            {
+                builder[i] = stringTable.Add(builder[i]);
+            }
+        }
+
+        public abstract bool TryGetDeclaredSymbolInfo(StringTable stringTable, SyntaxNode node, out DeclaredSymbolInfo declaredSymbolInfo);
+    }
+
+    internal abstract class AbstractSyntaxFactsService
+    {
         private readonly static ObjectPool<Stack<(SyntaxNodeOrToken nodeOrToken, bool leading, bool trailing)>> s_stackPool =
             new ObjectPool<Stack<(SyntaxNodeOrToken nodeOrToken, bool leading, bool trailing)>>(() => new Stack<(SyntaxNodeOrToken nodeOrToken, bool leading, bool trailing)>());
 
@@ -78,33 +134,6 @@ protected AbstractSyntaxFactsService()
         public abstract bool IsShebangDirectiveTrivia(SyntaxTrivia trivia);
         public abstract bool IsPreprocessorDirective(SyntaxTrivia trivia);
 
-        protected static List<Dictionary<string, string>> AllocateAliasMapList()
-            => s_aliasMapListPool.Allocate();
-
-        protected static void FreeAliasMapList(List<Dictionary<string, string>> list)
-        {
-            if (list != null)
-            {
-                foreach (var aliasMap in list)
-                {
-                    FreeAliasMap(aliasMap);
-                }
-
-                s_aliasMapListPool.ClearAndFree(list);
-            }
-        }
-
-        protected static void FreeAliasMap(Dictionary<string, string> aliasMap)
-        {
-            if (aliasMap != null)
-            {
-                s_aliasMapPool.ClearAndFree(aliasMap);
-            }
-        }
-
-        protected static Dictionary<string, string> AllocateAliasMap()
-            => s_aliasMapPool.Allocate();
-
         public bool IsOnSingleLine(SyntaxNode node, bool fullSpan)
         {
             // Use an actual Stack so we can write out deeply recursive structures without overflowing.
@@ -393,20 +422,5 @@ public string GetBannerText(SyntaxNode documentationCommentTriviaSyntax, Cancell
             => DocumentationCommentService.GetBannerText(documentationCommentTriviaSyntax, cancellationToken);
 
         protected abstract IDocumentationCommentService DocumentationCommentService { get; }
-
-        protected static void AppendTokens(SyntaxNode node, StringBuilder builder)
-        {
-            foreach (var child in node.ChildNodesAndTokens())
-            {
-                if (child.IsToken)
-                {
-                    builder.Append(child.AsToken().Text);
-                }
-                else
-                {
-                    AppendTokens(child.AsNode(), builder);
-                }
-            }
-        }
     }
 }

src/Workspaces/Core/Portable/LanguageServices/SyntaxFactsService/ISyntaxFactsService.cs

@@ -4,7 +4,6 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Threading;
-using Microsoft.CodeAnalysis.FindSymbols;
 using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Text;
 
@@ -226,8 +225,6 @@ internal interface ISyntaxFactsService : ILanguageService
         bool AreEquivalent(SyntaxToken token1, SyntaxToken token2);
         bool AreEquivalent(SyntaxNode node1, SyntaxNode node2);
 
-        bool TryGetDeclaredSymbolInfo(SyntaxNode node, out DeclaredSymbolInfo declaredSymbolInfo);
-
         string GetDisplayName(SyntaxNode node, DisplayNameOptions options, string rootNamespace = null);
 
         SyntaxNode GetContainingTypeDeclaration(SyntaxNode root, int position);

src/Workspaces/Core/Portable/Workspaces.csproj

@@ -177,6 +177,7 @@
     <Compile Include="..\..\..\Compilers\Core\Portable\InternalUtilities\StringExtensions.cs">
       <Link>InternalUtilities\StringExtensions.cs</Link>
     </Compile>
+    <Compile Include="..\..\..\Compilers\Core\Portable\InternalUtilities\StringTable.cs" Link="Utilities\StringTable.cs" />
     <Compile Include="..\..\..\Compilers\Core\Portable\InternalUtilities\TextChangeRangeExtensions.cs">
       <Link>InternalUtilities\TextChangeRangeExtensions.cs</Link>
     </Compile>

src/Workspaces/VisualBasic/Portable/BasicWorkspace.vbproj

@@ -166,6 +166,7 @@
     <Compile Include="Formatting\VisualBasicSyntaxFormattingService.vb" />
     <Compile Include="LanguageServices\VisualBasicCompilationFactoryService.vb" />
     <Compile Include="LanguageServices\VisualBasicCommandLineParserService.vb" />
+    <Compile Include="FindSymbols\VisualBasicDeclaredSymbolInfoFactoryService.vb" />
     <Compile Include="LanguageServices\VisualBasicDocumentationCommentService.vb" />
     <Compile Include="LanguageServices\VisualBasicSemanticFactsService.vb" />
     <Compile Include="LanguageServices\VisualBasicSymbolDeclarationService.vb" />