Make the SyntaxTreeIndex use a content based scheme for persisting data.

src/Workspaces/Core/Desktop/Workspace/Storage/PersistentStorageService.cs

@@ -17,7 +17,7 @@ namespace Microsoft.CodeAnalysis.Storage
     /// A service that enables storing and retrieving of information associated with solutions,
     /// projects or documents across runtime sessions.
     /// </summary>
-    internal abstract partial class AbstractPersistentStorageService : IPersistentStorageService
+    internal abstract partial class AbstractPersistentStorageService : IPersistentStorageService2
     {
         protected readonly IOptionService OptionService;
         private readonly SolutionSizeTracker _solutionSizeTracker;
@@ -58,8 +58,11 @@ protected AbstractPersistentStorageService(IOptionService optionService, bool te
         protected abstract bool ShouldDeleteDatabase(Exception exception);
 
         public IPersistentStorage GetStorage(Solution solution)
+            => GetStorage(solution, checkBranchId: true);
+
+        public IPersistentStorage GetStorage(Solution solution, bool checkBranchId)
         {
-            if (!ShouldUseDatabase(solution))
+            if (!ShouldUseDatabase(solution, checkBranchId))
             {
                 return NoOpPersistentStorage.Instance;
             }
@@ -136,16 +139,21 @@ private IPersistentStorage GetStorage(Solution solution, string workingFolderPat
             }
         }
 
-        private bool ShouldUseDatabase(Solution solution)
+        private bool ShouldUseDatabase(Solution solution, bool checkBranchId)
         {
             if (_testing)
             {
                 return true;
             }
 
-            // we only use database for primary solution. (Ex, forked solution will not use database)
-            if (solution.BranchId != solution.Workspace.PrimaryBranchId || solution.FilePath == null)
+            if (solution.FilePath == null)
+            {
+                return false;
+            }
+
+            if (checkBranchId && solution.BranchId != solution.Workspace.PrimaryBranchId)
             {
+                // we only use database for primary solution. (Ex, forked solution will not use database)
                 return false;
             }
 

src/Workspaces/Core/Portable/FindSymbols/SyntaxTree/SyntaxTreeIndex.cs

@@ -11,21 +11,21 @@ namespace Microsoft.CodeAnalysis.FindSymbols
 {
     internal sealed partial class SyntaxTreeIndex
     {
-        private readonly VersionStamp _version;
-
         private readonly LiteralInfo _literalInfo;
         private readonly IdentifierInfo _identifierInfo;
         private readonly ContextInfo _contextInfo;
         private readonly DeclarationInfo _declarationInfo;
 
         private SyntaxTreeIndex(
-            VersionStamp version,
+            Checksum textChecksum,
+            Checksum parseOptionsChecksum,
             LiteralInfo literalInfo,
             IdentifierInfo identifierInfo,
             ContextInfo contextInfo,
             DeclarationInfo declarationInfo)
         {
-            Version = version;
+            TextChecksum = textChecksum;
+            ParseOptionsChecksum = parseOptionsChecksum;
             _literalInfo = literalInfo;
             _identifierInfo = identifierInfo;
             _contextInfo = contextInfo;

src/Workspaces/Core/Portable/FindSymbols/SyntaxTree/SyntaxTreeIndex_Persistence.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
 using System;
-using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.CodeAnalysis.Shared.Extensions;
-using Roslyn.Utilities;
 using Microsoft.CodeAnalysis.Host;
-using Microsoft.CodeAnalysis.Versions;
+using Microsoft.CodeAnalysis.Serialization;
 using Microsoft.CodeAnalysis.Shared.Utilities;
+using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis.FindSymbols
 {
     internal sealed partial class SyntaxTreeIndex : IObjectWritable
     {
-        private const string PersistenceName = "<TreeInfoPersistence>";
-        private const string SerializationFormat = "6";
+        private const string PersistenceName = "<SyntaxTreeIndex>";
+        private const string SerializationFormat = "7";
 
-        /// <summary>
-        /// in memory cache will hold onto any info related to opened documents in primary branch or all documents in forked branch
-        /// 
-        /// this is not snapshot based so multiple versions of snapshots can re-use same data as long as it is relevant.
-        /// </summary>
-        private static readonly ConditionalWeakTable<BranchId, ConditionalWeakTable<DocumentId, SyntaxTreeIndex>> s_cache =
-            new ConditionalWeakTable<BranchId, ConditionalWeakTable<DocumentId, SyntaxTreeIndex>>();
+        public readonly Checksum TextChecksum;
+        public readonly Checksum ParseOptionsChecksum;
 
-        public readonly VersionStamp Version;
-
-        private void WriteVersion(ObjectWriter writer, string formatVersion)
+        private void WriteFormatAndChecksums(ObjectWriter writer, string formatVersion)
         {
             writer.WriteString(formatVersion);
-            this.Version.WriteTo(writer);
+            TextChecksum.WriteTo(writer);
+            ParseOptionsChecksum.WriteTo(writer);
         }
 
-        private static bool TryReadVersion(ObjectReader reader, string formatVersion, out VersionStamp version)
+        private static bool TryReadFormatAndChecksums(
+            ObjectReader reader, string formatVersion, 
+            out Checksum textChecksum, out Checksum parseOptionsChecksum)
         {
-            version = VersionStamp.Default;
+            textChecksum = null;
+            parseOptionsChecksum = null;
             if (reader.ReadString() != formatVersion)
             {
                 return false;
             }
 
-            version = VersionStamp.ReadFrom(reader);
+            textChecksum = Checksum.ReadFrom(reader);
+            parseOptionsChecksum = Checksum.ReadFrom(reader);
             return true;
         }
 
         private static async Task<SyntaxTreeIndex> LoadAsync(
             Document document, string persistenceName, string formatVersion,
-            Func<ObjectReader, VersionStamp, SyntaxTreeIndex> readFrom, CancellationToken cancellationToken)
+            Func<ObjectReader, Checksum, Checksum, SyntaxTreeIndex> readFrom, CancellationToken cancellationToken)
         {
-            var persistentStorageService = document.Project.Solution.Workspace.Services.GetService<IPersistentStorageService>();
-            var syntaxVersion = await document.GetSyntaxVersionAsync(cancellationToken).ConfigureAwait(false);
+            var solution = document.Project.Solution;
+            var persistentStorageService = (IPersistentStorageService2)solution.Workspace.Services.GetService<IPersistentStorageService>();
+
+            var (textChecksum, parseOptionsChecksum) = await GetChecksumsAsync(document, cancellationToken).ConfigureAwait(false);
 
             try
             {
                 // attempt to load from persisted state
-                using (var storage = persistentStorageService.GetStorage(document.Project.Solution))
+                using (var storage = persistentStorageService.GetStorage(solution, checkBranchId: false))
                 using (var stream = await storage.ReadStreamAsync(document, persistenceName, cancellationToken).ConfigureAwait(false))
                 using (var reader = ObjectReader.TryGetReader(stream))
                 {
                     if (reader != null)
                     {
-                        if (TryReadVersion(reader, formatVersion, out var persistVersion) &&
-                            document.CanReusePersistedSyntaxTreeVersion(syntaxVersion, persistVersion))
+                        if (DataPreambleMatches(reader, formatVersion, textChecksum, parseOptionsChecksum))
                         {
-                            return readFrom(reader, syntaxVersion);
+                            return readFrom(reader, textChecksum, parseOptionsChecksum);
                         }
                     }
                 }
@@ -77,20 +74,43 @@ private static bool TryReadVersion(ObjectReader reader, string formatVersion, ou
             return null;
         }
 
+        private static bool DataPreambleMatches(
+            ObjectReader reader, string formatVersion, Checksum textChecksum, Checksum parseOptionsChecksum)
+        {
+            return TryReadFormatAndChecksums(reader, formatVersion, out var persistTextChecksum, out var persistParseOptionsChecksum) &&
+                   persistTextChecksum == textChecksum &&
+                   persistParseOptionsChecksum == parseOptionsChecksum;
+        }
+
+        private static async Task<(Checksum textChecksum, Checksum parseOptionsChecksum)> GetChecksumsAsync(
+            Document document, CancellationToken cancellationToken)
+        {
+            var text = await document.GetTextAsync(cancellationToken).ConfigureAwait(false);
+            var textChecksum = Checksum.Create(WellKnownSynchronizationKinds.SourceText, text.GetChecksum());
+
+            var parseOptions = document.Project.ParseOptions;
+
+            var serializer = new Serializer(document.Project.Solution.Workspace);
+            var parseOptionsChecksum = ChecksumCache.GetOrCreate(
+                parseOptions, _ => serializer.CreateChecksum(parseOptions, cancellationToken));
+
+            return (textChecksum, parseOptionsChecksum);
+        }
+
         private static async Task<bool> SaveAsync(
             Document document, string persistenceName, string formatVersion, SyntaxTreeIndex data, CancellationToken cancellationToken)
         {
-            Contract.Requires(!await document.IsForkedDocumentWithSyntaxChangesAsync(cancellationToken).ConfigureAwait(false));
-
-            var persistentStorageService = document.Project.Solution.Workspace.Services.GetService<IPersistentStorageService>();
+            var solution = document.Project.Solution;
+            var persistentStorageService = (IPersistentStorageService2)solution.Workspace.Services.GetService<IPersistentStorageService>();
+            var (textChecksum, parseOptionsChecksum) = await GetChecksumsAsync(document, cancellationToken).ConfigureAwait(false);
 
             try
             {
-                using (var storage = persistentStorageService.GetStorage(document.Project.Solution))
+                using (var storage = persistentStorageService.GetStorage(solution, checkBranchId: false))
                 using (var stream = SerializableBytes.CreateWritableStream())
                 using (var writer = new ObjectWriter(stream, cancellationToken: cancellationToken))
                 {
-                    data.WriteVersion(writer, formatVersion);
+                    data.WriteFormatAndChecksums(writer, formatVersion);
                     data.WriteTo(writer);
 
                     stream.Position = 0;
@@ -105,24 +125,23 @@ private static bool TryReadVersion(ObjectReader reader, string formatVersion, ou
             return false;
         }
 
-        private static async Task<bool> PrecalculatedAsync(Document document, string persistenceName, string formatVersion, CancellationToken cancellationToken)
+        private static async Task<bool> PrecalculatedAsync(
+            Document document, string persistenceName, string formatVersion, CancellationToken cancellationToken)
         {
-            Contract.Requires(document.IsFromPrimaryBranch());
-
-            var persistentStorageService = document.Project.Solution.Workspace.Services.GetService<IPersistentStorageService>();
-            var syntaxVersion = await document.GetSyntaxVersionAsync(cancellationToken).ConfigureAwait(false);
+            var solution = document.Project.Solution;
+            var persistentStorageService = (IPersistentStorageService2)solution.Workspace.Services.GetService<IPersistentStorageService>();
+            var (textChecksum, parseOptionsChecksum) = await GetChecksumsAsync(document, cancellationToken).ConfigureAwait(false);
 
             // check whether we already have info for this document
             try
             {
-                using (var storage = persistentStorageService.GetStorage(document.Project.Solution))
+                using (var storage = persistentStorageService.GetStorage(solution, checkBranchId: false))
                 using (var stream = await storage.ReadStreamAsync(document, persistenceName, cancellationToken).ConfigureAwait(false))
                 using (var reader = ObjectReader.TryGetReader(stream))
                 {
                     if (reader != null)
                     {
-                        return TryReadVersion(reader, formatVersion, out var persistVersion) &&
-                               document.CanReusePersistedSyntaxTreeVersion(syntaxVersion, persistVersion);
+                        return DataPreambleMatches(reader, formatVersion, textChecksum, parseOptionsChecksum);
                     }
                 }
             }
@@ -142,7 +161,8 @@ public void WriteTo(ObjectWriter writer)
             _declarationInfo.WriteTo(writer);
         }
 
-        private static SyntaxTreeIndex ReadFrom(ObjectReader reader, VersionStamp version)
+        private static SyntaxTreeIndex ReadFrom(
+            ObjectReader reader, Checksum textChecksum, Checksum parseOptionsChecksum)
         {
             var literalInfo = LiteralInfo.TryReadFrom(reader);
             var identifierInfo = IdentifierInfo.TryReadFrom(reader);
@@ -155,118 +175,16 @@ private static SyntaxTreeIndex ReadFrom(ObjectReader reader, VersionStamp versio
             }
 
             return new SyntaxTreeIndex(
-                version, literalInfo.Value, identifierInfo.Value, contextInfo.Value, declarationInfo.Value);
+                textChecksum, parseOptionsChecksum, literalInfo.Value, identifierInfo.Value, contextInfo.Value, declarationInfo.Value);
         }
 
         private Task<bool> SaveAsync(Document document, CancellationToken cancellationToken)
-            => SaveAsync(document, s_cache, PersistenceName, SerializationFormat, cancellationToken);
-
-        private async Task<bool> SaveAsync(
-            Document document,
-            ConditionalWeakTable<BranchId, ConditionalWeakTable<DocumentId, SyntaxTreeIndex>> cache,
-            string persistenceName,
-            string serializationFormat,
-            CancellationToken cancellationToken)
-        {
-            var workspace = document.Project.Solution.Workspace;
-            var infoTable = GetInfoTable(document.Project.Solution.BranchId, workspace, cache);
-
-            // if it is forked document
-            if (await document.IsForkedDocumentWithSyntaxChangesAsync(cancellationToken).ConfigureAwait(false))
-            {
-                infoTable.Remove(document.Id);
-                infoTable.GetValue(document.Id, _ => this);
-                return false;
-            }
-
-            // okay, cache this info if it is from opened document or persistence failed.
-            var persisted = await SaveAsync(document, persistenceName, serializationFormat, this, cancellationToken).ConfigureAwait(false);
-            if (!persisted || document.IsOpen())
-            {
-                var primaryInfoTable = GetInfoTable(workspace.PrimaryBranchId, workspace, cache);
-                primaryInfoTable.Remove(document.Id);
-                primaryInfoTable.GetValue(document.Id, _ => this);
-            }
-
-            return persisted;
-        }
+            => SaveAsync(document, PersistenceName, SerializationFormat, this, cancellationToken);
 
         private static Task<SyntaxTreeIndex> LoadAsync(Document document, CancellationToken cancellationToken)
-            => LoadAsync(document, ReadFrom, s_cache, PersistenceName, SerializationFormat, cancellationToken);
-
-        private static async Task<SyntaxTreeIndex> LoadAsync(
-            Document document,
-            Func<ObjectReader, VersionStamp, SyntaxTreeIndex> reader,
-            ConditionalWeakTable<BranchId, ConditionalWeakTable<DocumentId, SyntaxTreeIndex>> cache,
-            string persistenceName,
-            string serializationFormat,
-            CancellationToken cancellationToken)
-        {
-            var infoTable = cache.GetValue(
-                document.Project.Solution.BranchId, 
-                _ => new ConditionalWeakTable<DocumentId, SyntaxTreeIndex>());
-            var version = await document.GetSyntaxVersionAsync(cancellationToken).ConfigureAwait(false);
-            // first look to see if we already have the info in the cache
-            if (infoTable.TryGetValue(document.Id, out var info) && info.Version == version)
-            {
-                return info;
-            }
-
-            // cache is invalid. remove it
-            infoTable.Remove(document.Id);
-
-            // check primary cache to see whether we have valid info there
-            var primaryInfoTable = cache.GetValue(
-                document.Project.Solution.Workspace.PrimaryBranchId,
-                _ => new ConditionalWeakTable<DocumentId, SyntaxTreeIndex>());
-            if (primaryInfoTable.TryGetValue(document.Id, out info) && info.Version == version)
-            {
-                return info;
-            }
-
-            // check whether we can get it from persistence service
-            info = await LoadAsync(document, persistenceName, serializationFormat, reader, cancellationToken).ConfigureAwait(false);
-            if (info != null)
-            {
-                // save it in the cache. persisted info is always from primary branch. no reason to save it to the branched document cache.
-                primaryInfoTable.Remove(document.Id);
-                primaryInfoTable.GetValue(document.Id, _ => info);
-                return info;
-            }
-
-            // well, we don't have this information.
-            return null;
-        }
+            => LoadAsync(document, PersistenceName, SerializationFormat, ReadFrom, cancellationToken);
 
         private static Task<bool> PrecalculatedAsync(Document document, CancellationToken cancellationToken)
             => PrecalculatedAsync(document, PersistenceName, SerializationFormat, cancellationToken);
-
-        private static ConditionalWeakTable<DocumentId, SyntaxTreeIndex> GetInfoTable(
-            BranchId branchId,
-            Workspace workspace,
-            ConditionalWeakTable<BranchId, ConditionalWeakTable<DocumentId, SyntaxTreeIndex>> cache)
-        {
-            return cache.GetValue(branchId, id =>
-            {
-                if (id == workspace.PrimaryBranchId)
-                {
-                    workspace.DocumentClosed += (sender, e) =>
-                    {
-                        if (!e.Document.IsFromPrimaryBranch())
-                        {
-                            return;
-                        }
-
-                        if (cache.TryGetValue(e.Document.Project.Solution.BranchId, out var infoTable))
-                        {
-                            // remove closed document from primary branch from live cache.
-                            infoTable.Remove(e.Document.Id);
-                        }
-                    };
-                }
-
-                return new ConditionalWeakTable<DocumentId, SyntaxTreeIndex>();
-            });
-        }
     }
 }
\ No newline at end of file

src/Workspaces/Core/Portable/Workspace/Host/PersistentStorage/IPersistentStorageService.cs

@@ -9,4 +9,9 @@ public interface IPersistentStorageService : IWorkspaceService
     {
         IPersistentStorage GetStorage(Solution solution);
     }
+
+    internal interface IPersistentStorageService2 : IPersistentStorageService
+    {
+        IPersistentStorage GetStorage(Solution solution, bool checkBranchId);
+    }
 }
\ No newline at end of file

src/Workspaces/Core/Portable/Workspace/Host/PersistentStorage/NoOpPersistentStorageService.cs

@@ -2,7 +2,7 @@
 
 namespace Microsoft.CodeAnalysis.Host
 {
-    internal class NoOpPersistentStorageService : IPersistentStorageService
+    internal class NoOpPersistentStorageService : IPersistentStorageService2
     {
         public static readonly IPersistentStorageService Instance = new NoOpPersistentStorageService();
 
@@ -12,5 +12,8 @@ private NoOpPersistentStorageService()
 
         public IPersistentStorage GetStorage(Solution solution)
             => NoOpPersistentStorage.Instance;
+
+        public IPersistentStorage GetStorage(Solution solution, bool checkBranchId)
+            => NoOpPersistentStorage.Instance;
     }
 }
\ No newline at end of file

src/Workspaces/CoreTest/Host/WorkspaceServices/TestPersistenceService.cs

@@ -7,9 +7,12 @@
 namespace Microsoft.CodeAnalysis.UnitTests.Persistence
 {
     [ExportWorkspaceService(typeof(IPersistentStorageService), "Test"), Shared]
-    public class TestPersistenceService : IPersistentStorageService
+    public class TestPersistenceService : IPersistentStorageService2
     {
         public IPersistentStorage GetStorage(Solution solution)
             => NoOpPersistentStorage.Instance;
+
+        public IPersistentStorage GetStorage(Solution solution, bool checkBranchId)
+            => NoOpPersistentStorage.Instance;
     }
 }
\ No newline at end of file