made document checksum not to use persistant service.

we found 2 issues. 1. hard to validate persisted data and 2. recalculating checksum is faster than reading from storage.

src/VisualStudio/Core/Test.Next/Services/AssetServiceTests.cs

@@ -8,6 +8,7 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Editor.UnitTests.Workspaces;
 using Microsoft.CodeAnalysis.Remote;
+using Microsoft.CodeAnalysis.Remote.DebugUtil;
 using Roslyn.Test.Utilities;
 using Roslyn.Utilities;
 using Roslyn.VisualStudio.Next.UnitTests.Mocks;

src/VisualStudio/Core/Test.Next/Services/ServiceHubServicesTests.cs

@@ -12,6 +12,7 @@
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Editor.UnitTests.Workspaces;
 using Microsoft.CodeAnalysis.Remote;
+using Microsoft.CodeAnalysis.Remote.DebugUtil;
 using Microsoft.CodeAnalysis.Text;
 using Microsoft.CodeAnalysis.TodoComments;
 using Roslyn.Test.Utilities;

src/VisualStudio/Core/Test.Next/Services/SolutionServiceTests.cs

@@ -9,6 +9,7 @@
 using Microsoft.CodeAnalysis.Editor.UnitTests.Workspaces;
 using Microsoft.CodeAnalysis.Options;
 using Microsoft.CodeAnalysis.Remote;
+using Microsoft.CodeAnalysis.Remote.DebugUtil;
 using Microsoft.CodeAnalysis.SolutionCrawler;
 using Microsoft.CodeAnalysis.Text;
 using Microsoft.VisualStudio.LanguageServices.Remote;

src/VisualStudio/Core/Test.Next/VisualStudioTest.Next.csproj

@@ -263,7 +263,6 @@
     <Compile Include="Mocks\TestOptionSet.cs" />
     <Compile Include="Remote\JsonConverterTests.cs" />
     <Compile Include="Remote\RemoteHostClientServiceFactoryTests.cs" />
-    <Compile Include="TestUtils.cs" />
     <Compile Include="Services\SolutionServiceTests.cs" />
     <Compile Include="Services\AssetStorageTests.cs" />
     <Compile Include="Services\AssetServiceTests.cs" />

src/Workspaces/Core/Portable/Log/FunctionId.cs

@@ -327,50 +327,69 @@ internal enum FunctionId
         BKTree_ExceptionInCacheRead,
         IntellisenseBuild_Failed,
 
+        FileTextLoader_FileLengthThresholdExceeded,
+
         // Generic performance measurement action IDs
         MeasurePerformance_StartAction,
         MeasurePerformance_StopAction,
 
-        RemoteHostClientService_AddGlobalAssetsAsync,
-        RemoteHostClientService_RemoveGlobalAssets,
-        RemoteHostClientService_Enabled,
-        ServiceHubRemoteHostClient_CreateAsync,
-        SolutionSynchronizationServiceFactory_CreatePinnedRemotableDataScopeAsync,
-        PinnedRemotableDataScope_GetRemotableData,
         Serializer_CreateChecksum,
         Serializer_Serialize,
         Serializer_Deserialize,
+
         CodeAnalysisService_CalculateDiagnosticsAsync,
         CodeAnalysisService_SerializeDiagnosticResultAsync,
-        AssetStorage_CleanAssets,
-        AssetService_GetAssetAsync,
-        SnapshotService_RequestAssetAsync,
-        CompilationService_GetCompilationAsync,
-        RemoteHostService_SynchronizePrimaryWorkspaceAsync,
-        RemoteHostService_SynchronizeGlobalAssetsAsync,
-        AssetStorage_TryGetAsset,
-        AssetService_SynchronizeAssetsAsync,
-        AssetService_SynchronizeSolutionAssetsAsync,
         CodeAnalysisService_GetReferenceCountAsync,
         CodeAnalysisService_FindReferenceLocationsAsync,
         CodeAnalysisService_FindReferenceMethodsAsync,
         CodeAnalysisService_GetFullyQualifiedName,
-        SolutionChecksumUpdater_SynchronizePrimaryWorkspace,
-        SolutionState_ComputeChecksumsAsync,
-        ProjectState_ComputeChecksumsAsync,
-        DocumentState_ComputeChecksumsAsync,
-        JsonRpcSession_RequestAssetAsync,
-        SolutionSynchronizationService_GetRemotableData,
-        AssetService_SynchronizeProjectAssetsAsync,
-        FileTextLoader_FileLengthThresholdExceeded,
+        CodeAnalysisService_GetTodoCommentsAsync,
+        CodeAnalysisService_GetDesignerAttributesAsync,
+
+        ServiceHubRemoteHostClient_CreateAsync,
+        PinnedRemotableDataScope_GetRemotableData,
+
         RemoteHost_Connect,
         RemoteHost_Disconnect,
-        CodeAnalysisService_GetTodoCommentsAsync,
+
+        RemoteHostClientService_AddGlobalAssetsAsync,
+        RemoteHostClientService_RemoveGlobalAssets,
+        RemoteHostClientService_Enabled,
+        RemoteHostClientService_Restarted,
+
+        RemoteHostService_SynchronizePrimaryWorkspaceAsync,
+        RemoteHostService_SynchronizeGlobalAssetsAsync,
+
+        AssetStorage_CleanAssets,
+        AssetStorage_TryGetAsset,
+
+        AssetService_GetAssetAsync,
+        AssetService_SynchronizeAssetsAsync,
+        AssetService_SynchronizeSolutionAssetsAsync,
+        AssetService_SynchronizeProjectAssetsAsync,
+
         CodeLens_GetReferenceCountAsync,
         CodeLens_FindReferenceLocationsAsync,
         CodeLens_FindReferenceMethodsAsync,
         CodeLens_GetFullyQualifiedName,
-        RemoteHostClientService_Restarted,
-        CodeAnalysisService_GetDesignerAttributesAsync,
+
+        SolutionState_ComputeChecksumsAsync,
+        ProjectState_ComputeChecksumsAsync,
+        DocumentState_ComputeChecksumsAsync,
+
+        SolutionSynchronizationService_GetRemotableData,
+        SolutionSynchronizationServiceFactory_CreatePinnedRemotableDataScopeAsync,
+
+        SolutionChecksumUpdater_SynchronizePrimaryWorkspace,
+
+        JsonRpcSession_RequestAssetAsync,
+
+        SolutionService_GetSolutionAsync,
+        SolutionService_UpdatePrimaryWorkspaceAsync,
+
+        SnapshotService_RequestAssetAsync,
+
+        CompilationService_GetCompilationAsync,
+        SolutionCreator_AssetDifferences,
     }
 }

src/Workspaces/Core/Portable/Workspace/Solution/TextDocumentState_Checksum.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Internal.Log;
 using Microsoft.CodeAnalysis.Serialization;
-using Microsoft.CodeAnalysis.Shared.Utilities;
-using Microsoft.CodeAnalysis.Versions;
-using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis
 {
     internal partial class TextDocumentState
     {
-        private const string TextChecksum = nameof(TextChecksum);
-        private const string SerializationFormat = "1";
-
         public bool TryGetStateChecksums(out DocumentStateChecksums stateChecksums)
         {
             return _lazyChecksums.TryGetValue(out stateChecksums);
@@ -42,85 +34,10 @@ private async Task<DocumentStateChecksums> ComputeChecksumsAsync(CancellationTok
                 var serializer = new Serializer(solutionServices.Workspace);
 
                 var infoChecksum = serializer.CreateChecksum(Info.Attributes, cancellationToken);
-                var textChecksum = await GetTextChecksumAsync(serializer, await textAndVersionTask.ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
+                var textChecksum = serializer.CreateChecksum((await textAndVersionTask.ConfigureAwait(false)).Text, cancellationToken);
 
                 return new DocumentStateChecksums(infoChecksum, textChecksum);
             }
         }
-
-        private async Task<Checksum> GetTextChecksumAsync(Serializer serializer, TextAndVersion textAndVersion, CancellationToken cancellationToken)
-        {
-            // calculating checksum for source text is one of most expansive checksum calculation we need to do.
-            // this should let us get around it if possible
-            var solution = solutionServices.Workspace.CurrentSolution;
-
-            var document = solution.GetDocument(Id);
-            if (document == null)
-            {
-                // can't use persistent service since it is based on solution objects
-                return serializer.CreateChecksum(textAndVersion.Text, cancellationToken);
-            }
-
-            var storage = solution.Workspace.Services.GetService<IPersistentStorageService>()?.GetStorage(solution);
-            if (storage == null)
-            {
-                // persistent service not available
-                return serializer.CreateChecksum(textAndVersion.Text, cancellationToken);
-            }
-
-            try
-            {
-                using (var stream = await storage.ReadStreamAsync(document, TextChecksum, cancellationToken).ConfigureAwait(false))
-                using (var reader = ObjectReader.TryGetReader(stream))
-                {
-                    if (TryReadVersion(reader, out var persistedVersion) &&
-                        document.CanReusePersistedTextVersion(textAndVersion.Version, persistedVersion))
-                    {
-                        return Checksum.ReadFrom(reader);
-                    }
-                }
-
-                // either checksum doesn't exist or can't reuse. re-calculate the checksum
-                var checksum = serializer.CreateChecksum(textAndVersion.Text, cancellationToken);
-
-                // save newly calculated checksum
-                using (var stream = SerializableBytes.CreateWritableStream())
-                using (var writer = new ObjectWriter(stream, cancellationToken))
-                {
-                    WriteVersionsTo(writer, textAndVersion.Version);
-                    checksum.WriteTo(writer);
-
-                    stream.Position = 0;
-                    await storage.WriteStreamAsync(document, TextChecksum, stream, cancellationToken).ConfigureAwait(false);
-                }
-
-                return checksum;
-            }
-            catch (Exception e) when (IOUtilities.IsNormalIOException(e))
-            {
-                // Storage APIs can throw arbitrary exceptions.
-            }
-
-            // go simple route if persistent thing didn't work out
-            return serializer.CreateChecksum(textAndVersion.Text, cancellationToken);
-        }
-
-        private void WriteVersionsTo(ObjectWriter writer, VersionStamp version)
-        {
-            writer.WriteString(SerializationFormat);
-            version.WriteTo(writer); ;
-        }
-
-        private static bool TryReadVersion(ObjectReader reader, out VersionStamp persistedVersion)
-        {
-            persistedVersion = VersionStamp.Default;
-            if (reader?.ReadString() != SerializationFormat)
-            {
-                return false;
-            }
-
-            persistedVersion = VersionStamp.ReadFrom(reader);
-            return true;
-        }
     }
 }

src/Workspaces/Remote/Core/RemoteWorkspaces.csproj

@@ -78,6 +78,7 @@
     <Compile Include="Services\TemporaryWorkspace.cs" />
     <Compile Include="Services\TemporaryWorkspaceOptionsServiceFactory.cs" />
     <Compile Include="Storage\RemotePersistentStorageLocationService.cs" />
+    <Compile Include="TestUtils.cs" />
   </ItemGroup>
   <Import Project="..\..\..\..\build\Targets\Imports.targets" />
 </Project>
\ No newline at end of file

src/Workspaces/Remote/Core/Services/SolutionCreator.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
-using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.Internal.Log;
 using Microsoft.CodeAnalysis.Serialization;
 using Microsoft.CodeAnalysis.Text;
+using Microsoft.CodeAnalysis.Remote.DebugUtil;
 using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis.Remote
@@ -79,10 +80,8 @@ public async Task<Solution> CreateSolutionAsync(Checksum newSolutionChecksum)
                 solution = await UpdateProjectsAsync(solution, oldSolutionChecksums.Projects, newSolutionChecksums.Projects).ConfigureAwait(false);
             }
 
-#if DEBUG
             // make sure created solution has same checksum as given one
-            Contract.Requires(newSolutionChecksum == await solution.State.GetChecksumAsync(_cancellationToken).ConfigureAwait(false));
-#endif
+            await ValidateChecksumAsync(newSolutionChecksum, solution).ConfigureAwait(false);
 
             return solution;
         }
@@ -613,5 +612,64 @@ private ImmutableArray<string> GetStrongNameKeyPaths(ProjectInfo.ProjectAttribut
 
             return builder.ToImmutableAndFree();
         }
+
+        private async Task ValidateChecksumAsync(Checksum givenSolutionChecksum, Solution solution)
+        {
+            // have this to avoid error on async
+            await SpecializedTasks.EmptyTask;
+
+#if DEBUG
+            var currentSolutionChecksum = await solution.State.GetChecksumAsync(_cancellationToken).ConfigureAwait(false);
+
+            if (givenSolutionChecksum == currentSolutionChecksum)
+            {
+                return;
+            }
+
+            Contract.Requires(false, "checksum not same");
+
+            var map = solution.GetAssetMap();
+            await RemoveDuplicateChecksumsAsync(givenSolutionChecksum, map).ConfigureAwait(false);
+
+            foreach (var kv in map.Where(kv => kv.Value is ChecksumWithChildren).ToList())
+            {
+                map.Remove(kv.Key);
+            }
+
+            var sb = new StringBuilder();
+            foreach (var kv in map)
+            {
+                sb.AppendLine($"{kv.Key.ToString()}, {kv.Value.ToString()}");
+            }
+
+            Logger.Log(FunctionId.SolutionCreator_AssetDifferences, sb.ToString());
+#endif
+
+            return;
+        }
+
+        private async Task RemoveDuplicateChecksumsAsync(Checksum givenSolutionChecksum, Dictionary<Checksum, object> map)
+        {
+            var solutionChecksums = await _assetService.GetAssetAsync<SolutionStateChecksums>(givenSolutionChecksum, _cancellationToken).ConfigureAwait(false);
+            map.RemoveChecksums(solutionChecksums);
+
+            foreach (var projectChecksum in solutionChecksums.Projects)
+            {
+                var projectChecksums = await _assetService.GetAssetAsync<ProjectStateChecksums>(projectChecksum, _cancellationToken).ConfigureAwait(false);
+                map.RemoveChecksums(projectChecksums);
+
+                foreach (var documentChecksum in projectChecksums.Documents)
+                {
+                    var documentChecksums = await _assetService.GetAssetAsync<DocumentStateChecksums>(documentChecksum, _cancellationToken).ConfigureAwait(false);
+                    map.RemoveChecksums(documentChecksums);
+                }
+
+                foreach (var documentChecksum in projectChecksums.AdditionalDocuments)
+                {
+                    var documentChecksums = await _assetService.GetAssetAsync<DocumentStateChecksums>(documentChecksum, _cancellationToken).ConfigureAwait(false);
+                    map.RemoveChecksums(documentChecksums);
+                }
+            }
+        }
     }
 }

src/Workspaces/Remote/Core/TestUtils.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading;
+using Microsoft.CodeAnalysis.Serialization;
+using Roslyn.Utilities;
+
+namespace Microsoft.CodeAnalysis.Remote.DebugUtil
+{
+    internal static class TestUtils
+    {
+        public static void RemoveChecksums(this Dictionary<Checksum, object> map, ChecksumWithChildren checksums)
+        {
+            map.Remove(checksums.Checksum);
+
+            foreach (var child in checksums.Children)
+            {
+                var checksum = child as Checksum;
+                if (checksum != null)
+                {
+                    map.Remove(checksum);
+                }
+
+                var collection = child as ChecksumCollection;
+                if (collection != null)
+                {
+                    foreach (var item in collection)
+                    {
+                        map.Remove(item);
+                    }
+                }
+            }
+        }
+
+        public static Dictionary<Checksum, object> GetAssetMap(this Solution solution)
+        {
+            var map = new Dictionary<Checksum, object>();
+
+            AppendAssetMap(solution, map);
+
+            return map;
+        }
+
+        public static Dictionary<Checksum, object> GetAssetMap(this Project project)
+        {
+            var map = new Dictionary<Checksum, object>();
+
+            AppendAssetMap(project, map);
+
+            return map;
+        }
+
+        public static void AppendAssetMap(this Solution solution, Dictionary<Checksum, object> map)
+        {
+            SolutionStateChecksums solutionChecksums;
+            Contract.Requires(solution.State.TryGetStateChecksums(out solutionChecksums));
+
+            solutionChecksums.Find(solution.State, Flatten(solutionChecksums), map, CancellationToken.None);
+
+            foreach (var project in solution.Projects)
+            {
+                AppendAssetMap(project, map);
+            }
+        }
+
+        private static void AppendAssetMap(Project project, Dictionary<Checksum, object> map)
+        {
+            ProjectStateChecksums projectChecksums;
+            if (!project.State.TryGetStateChecksums(out projectChecksums))
+            {
+                Contract.Requires(!RemoteSupportedLanguages.IsSupported(project.Language));
+                return;
+            }
+
+            projectChecksums.Find(project.State, Flatten(projectChecksums), map, CancellationToken.None);
+
+            foreach (var document in project.Documents)
+            {
+                AppendAssetMap(document, map);
+            }
+
+            foreach (var document in project.AdditionalDocuments)
+            {
+                AppendAssetMap(document, map);
+            }
+        }
+
+        private static void AppendAssetMap(TextDocument document, Dictionary<Checksum, object> map)
+        {
+            DocumentStateChecksums documentChecksums;
+            Contract.Requires(document.State.TryGetStateChecksums(out documentChecksums));
+
+            documentChecksums.Find(document.State, Flatten(documentChecksums), map, CancellationToken.None);
+
+            // fix up due to source text can't be obtained synchronously in product code
+            map[documentChecksums.Text] = document.State.GetTextSynchronously(CancellationToken.None);
+        }
+
+        private static HashSet<Checksum> Flatten(ChecksumWithChildren checksums)
+        {
+            var set = new HashSet<Checksum>();
+            set.Add(checksums.Checksum);
+
+            foreach (var child in checksums.Children)
+            {
+                var checksum = child as Checksum;
+                if (checksum != null)
+                {
+                    set.Add(checksum);
+                }
+
+                var collection = child as ChecksumCollection;
+                if (collection != null)
+                {
+                    foreach (var item in collection)
+                    {
+                        set.Add(item);
+                    }
+                }
+            }
+
+            return set;
+        }
+    }
+}