Skip to content

R
roslyn

lwm1986 / roslyn
与 Fork 源项目一致

从无法访问的项目Fork

通知 1
Star 0
Fork 0
R
roslyn

体验新版 GitCode，发现更多精彩内容 >>

提交 3f487e01 编写于 作者: C CyrusNajmabadi 提交者： Heejae Chang
浏览文件
操作

Store checksums as a first-class piece of data in our persistence layer. (#31538) 

* Store checksums as a first-class piece of data in our persistence layer.

* Include the format version in the persistence checksum.

* Move symbol tree over to the checksym approach.

* Update sql persistence service to use checksums.

* Update tests.

* Fix wrapper.

* Remove projects.

* Add tests with checksums.

* Add more tests.

* Add test.

* Add comments.

* Simplify.

* Simplify.

* Revert.

* Revert.

* Revert.

* Add comment.

* Add comment.

* Add comment.

* Rev the checksum.

* Add comment.

* Add comment.

* Add comment.

* Add comment.

* Simplify.

* merge

* rename

* No allow checksym creation

* Rename

* cache strings

* simplify

* Remove optional params

* Update src/Workspaces/Core/Portable/FindSymbols/SyntaxTree/SyntaxTreeIndex_Persistence.cs
Co-Authored-By: NAndrew Harper <aharper@breuer.com>
上级 9b888310
展开全部 隐藏空白更改
内联 并排
Showing with 712 addition and 229 deletion
src/VisualStudio/CSharp/Test/PersistentStorage/SQLitePersistentStorageTests.cs
浏览文件 @ 3f487e01
......@@ -15,7 +15,7 @@ namespace Microsoft.CodeAnalysis.UnitTests.WorkspaceServices
/// </remarks>
public class SQLitePersistentStorageTests : AbstractPersistentStorageTests
{
internal override IPersistentStorageService GetStorageService(IPersistentStorageLocationService locationService, ISolutionSizeTracker solutionSizeTracker, IPersistentStorageFaultInjector faultInjector)
internal override IChecksummedPersistentStorageService GetStorageService(IPersistentStorageLocationService locationService, ISolutionSizeTracker solutionSizeTracker, IPersistentStorageFaultInjector faultInjector)
=> new SQLitePersistentStorageService(_persistentEnabledOptionService, locationService, solutionSizeTracker, faultInjector);
[Fact]
......
src/Workspaces/Core/Portable/FindSymbols/SymbolTree/SymbolTreeInfo_Metadata.cs
浏览文件 @ 3f487e01
......@@ -169,7 +169,13 @@ private static Checksum GetMetadataChecksumSlow(Solution solution, PortableExecu
{
var serializer = solution.Workspace.Services.GetService<ISerializerService>();
var checksum = serializer.CreateChecksum(reference, cancellationToken);
return checksum;
// Include serialization format version in our checksum. That way if the
// version ever changes, all persisted data won't match the current checksum
// we expect, and we'll recompute things.
return Checksum.Create(
WellKnownSynchronizationKind.SymbolTreeInfo,
new[] { checksum, SerializationFormatChecksum });
});
}
......@@ -188,7 +194,7 @@ private static Checksum GetMetadataChecksumSlow(Solution solution, PortableExecu
loadOnly,
createAsync: () => CreateMetadataSymbolTreeInfoAsync(solution, checksum, reference, cancellationToken),
keySuffix: "_Metadata_" + filePath,
tryReadObject: reader => TryReadSymbolTreeInfo(reader, (names, nodes) => GetSpellCheckerTask(solution, checksum, filePath, names, nodes)),
tryReadObject: reader => TryReadSymbolTreeInfo(reader, checksum, (names, nodes) => GetSpellCheckerTask(solution, checksum, filePath, names, nodes)),
cancellationToken: cancellationToken);
Contract.ThrowIfFalse(result != null || loadOnly == true, "Result can only be null if 'loadOnly: true' was passed.");
return result;
......
src/Workspaces/Core/Portable/FindSymbols/SymbolTree/SymbolTreeInfo_Serialization.cs
浏览文件 @ 3f487e01
......@@ -2,6 +2,7 @@
using System;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
......@@ -19,7 +20,7 @@ namespace Microsoft.CodeAnalysis.FindSymbols
internal partial class SymbolTreeInfo : IObjectWritable
{
private const string PrefixMetadataSymbolTreeInfo = "<SymbolTreeInfo>";
private const string SerializationFormat = "17";
private static readonly Checksum SerializationFormatChecksum = Checksum.Create("18");
/// <summary>
/// Loads the SpellChecker for a given assembly symbol (metadata or project). If the
......@@ -65,14 +66,14 @@ internal partial class SymbolTreeInfo : IObjectWritable
}
// Ok, we can use persistence. First try to load from the persistence service.
var persistentStorageService = (IPersistentStorageService2)solution.Workspace.Services.GetService<IPersistentStorageService>();
var persistentStorageService = (IChecksummedPersistentStorageService)solution.Workspace.Services.GetService<IPersistentStorageService>();
T result;
using (var storage = persistentStorageService.GetStorage(solution, checkBranchId: false))
{
// Get the unique key to identify our data.
var key = PrefixMetadataSymbolTreeInfo + keySuffix;
using (var stream = await storage.ReadStreamAsync(key, cancellationToken).ConfigureAwait(false))
using (var stream = await storage.ReadStreamAsync(key, checksum, cancellationToken).ConfigureAwait(false))
using (var reader = ObjectReader.TryGetReader(stream))
{
if (reader != null)
......@@ -81,8 +82,11 @@ internal partial class SymbolTreeInfo : IObjectWritable
// If we're able to, and the version of the persisted data matches
// our version, then we can reuse this instance.
result = tryReadObject(reader);
if (result?.Checksum == checksum)
if (result != null)
{
// If we were able to read something in, it's checksum better
// have matched the checksum we expected.
Debug.Assert(result.Checksum == checksum);
return result;
}
}
......@@ -108,7 +112,7 @@ internal partial class SymbolTreeInfo : IObjectWritable
result.WriteTo(writer);
stream.Position = 0;
await storage.WriteStreamAsync(key, stream, cancellationToken).ConfigureAwait(false);
await storage.WriteStreamAsync(key, stream, checksum, cancellationToken).ConfigureAwait(false);
}
}
......@@ -128,9 +132,6 @@ async Task<T> CreateWithLoggingAsync()
public void WriteTo(ObjectWriter writer)
{
writer.WriteString(SerializationFormat);
Checksum.WriteTo(writer);
writer.WriteString(_concatenatedNames);
writer.WriteInt32(_nodes.Length);
......@@ -157,53 +158,48 @@ public void WriteTo(ObjectWriter writer)
internal static SymbolTreeInfo ReadSymbolTreeInfo_ForTestingPurposesOnly(
ObjectReader reader, Checksum checksum)
{
return TryReadSymbolTreeInfo(reader,
return TryReadSymbolTreeInfo(reader, checksum,
(names, nodes) => Task.FromResult(
new SpellChecker(checksum, nodes.Select(n => new StringSlice(names, n.NameSpan)))));
}
private static SymbolTreeInfo TryReadSymbolTreeInfo(
ObjectReader reader,
Checksum checksum,
Func<string, ImmutableArray<Node>, Task<SpellChecker>> createSpellCheckerTask)
{
try
{
var formatVersion = reader.ReadString();
if (string.Equals(formatVersion, SerializationFormat, StringComparison.Ordinal))
{
var checksum = Checksum.ReadFrom(reader);
var concatenatedNames = reader.ReadString();
var concatenatedNames = reader.ReadString();
var nodeCount = reader.ReadInt32();
var nodes = ArrayBuilder<Node>.GetInstance(nodeCount);
for (var i = 0; i < nodeCount; i++)
{
var start = reader.ReadInt32();
var length = reader.ReadInt32();
var parentIndex = reader.ReadInt32();
var nodeCount = reader.ReadInt32();
var nodes = ArrayBuilder<Node>.GetInstance(nodeCount);
for (var i = 0; i < nodeCount; i++)
{
var start = reader.ReadInt32();
var length = reader.ReadInt32();
var parentIndex = reader.ReadInt32();
nodes.Add(new Node(new TextSpan(start, length), parentIndex));
}
nodes.Add(new Node(new TextSpan(start, length), parentIndex));
}
var inheritanceMap = new OrderPreservingMultiDictionary<int, int>();
var inheritanceMapKeyCount = reader.ReadInt32();
for (var i = 0; i < inheritanceMapKeyCount; i++)
{
var key = reader.ReadInt32();
var valueCount = reader.ReadInt32();
var inheritanceMap = new OrderPreservingMultiDictionary<int, int>();
var inheritanceMapKeyCount = reader.ReadInt32();
for (var i = 0; i < inheritanceMapKeyCount; i++)
for (var j = 0; j < valueCount; j++)
{
var key = reader.ReadInt32();
var valueCount = reader.ReadInt32();
for (var j = 0; j < valueCount; j++)
{
var value = reader.ReadInt32();
inheritanceMap.Add(key, value);
}
var value = reader.ReadInt32();
inheritanceMap.Add(key, value);
}
var nodeArray = nodes.ToImmutableAndFree();
var spellCheckerTask = createSpellCheckerTask(concatenatedNames, nodeArray);
return new SymbolTreeInfo(checksum, concatenatedNames, nodeArray, spellCheckerTask, inheritanceMap);
}
var nodeArray = nodes.ToImmutableAndFree();
var spellCheckerTask = createSpellCheckerTask(concatenatedNames, nodeArray);
return new SymbolTreeInfo(checksum, concatenatedNames, nodeArray, spellCheckerTask, inheritanceMap);
}
catch
{
......
src/Workspaces/Core/Portable/FindSymbols/SymbolTree/SymbolTreeInfo_Source.cs
浏览文件 @ 3f487e01
......@@ -35,7 +35,7 @@ private static void FreeSymbolMap(MultiDictionary<string, ISymbol> symbolMap)
loadOnly: false,
createAsync: () => CreateSourceSymbolTreeInfoAsync(project, checksum, cancellationToken),
keySuffix: "_Source_" + project.FilePath,
tryReadObject: reader => TryReadSymbolTreeInfo(reader, (names, nodes) => GetSpellCheckerTask(project.Solution, checksum, project.FilePath, names, nodes)),
tryReadObject: reader => TryReadSymbolTreeInfo(reader, checksum, (names, nodes) => GetSpellCheckerTask(project.Solution, checksum, project.FilePath, names, nodes)),
cancellationToken: cancellationToken);
Contract.ThrowIfNull(result, "Result should never be null as we passed 'loadOnly: false'.");
return result;
......@@ -86,6 +86,11 @@ private static async Task<Checksum> ComputeSourceSymbolsChecksumAsync(ProjectSta
allChecksums.Add(compilationOptionsChecksum);
allChecksums.Add(parseOptionsChecksum);
// Include serialization format version in our checksum. That way if the
// version ever changes, all persisted data won't match the current checksum
// we expect, and we'll recompute things.
allChecksums.Add(SerializationFormatChecksum);
var checksum = Checksum.Create(WellKnownSynchronizationKind.SymbolTreeInfo, allChecksums);
return checksum;
}
......
src/Workspaces/Core/Portable/FindSymbols/SyntaxTree/SyntaxTreeIndex_Persistence.cs
浏览文件 @ 3f487e01
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.Host;
......@@ -13,48 +15,26 @@ namespace Microsoft.CodeAnalysis.FindSymbols
internal sealed partial class SyntaxTreeIndex : IObjectWritable
{
private const string PersistenceName = "<SyntaxTreeIndex>";
private const string SerializationFormat = "15";
private static readonly Checksum SerializationFormatChecksum = Checksum.Create("16");
public readonly Checksum Checksum;
private void WriteFormatAndChecksum(ObjectWriter writer, string formatVersion)
{
writer.WriteString(formatVersion);
Checksum.WriteTo(writer);
}
private static bool TryReadFormatAndChecksum(
ObjectReader reader, string formatVersion, out Checksum checksum)
{
checksum = null;
if (reader.ReadString() != formatVersion)
{
return false;
}
checksum = Checksum.ReadFrom(reader);
return true;
}
private static async Task<SyntaxTreeIndex> LoadAsync(
Document document, Checksum checksum, CancellationToken cancellationToken)
{
var solution = document.Project.Solution;
var persistentStorageService = (IPersistentStorageService2)solution.Workspace.Services.GetService<IPersistentStorageService>();
var persistentStorageService = (IChecksummedPersistentStorageService)solution.Workspace.Services.GetService<IPersistentStorageService>();
try
{
// attempt to load from persisted state
using (var storage = persistentStorageService.GetStorage(solution, checkBranchId: false))
using (var stream = await storage.ReadStreamAsync(document, PersistenceName, cancellationToken).ConfigureAwait(false))
using (var stream = await storage.ReadStreamAsync(document, PersistenceName, checksum, cancellationToken).ConfigureAwait(false))
using (var reader = ObjectReader.TryGetReader(stream))
{
if (reader != null)
{
if (FormatAndChecksumMatches(reader, SerializationFormat, checksum))
{
return ReadFrom(GetStringTable(document.Project), reader, checksum);
}
return ReadFrom(GetStringTable(document.Project), reader, checksum);
}
}
}
......@@ -66,13 +46,6 @@ private void WriteFormatAndChecksum(ObjectWriter writer, string formatVersion)
return null;
}
private static bool FormatAndChecksumMatches(
ObjectReader reader, string formatVersion, Checksum checksum)
{
return TryReadFormatAndChecksum(reader, formatVersion, out var persistChecksum) &&
persistChecksum == checksum;
}
public static async Task<Checksum> GetChecksumAsync(
Document document, CancellationToken cancellationToken)
{
......@@ -80,20 +53,25 @@ private void WriteFormatAndChecksum(ObjectWriter writer, string formatVersion)
// any time the SyntaxTree could have changed. Right now, that can only happen if the
// text of the document changes, or the ParseOptions change. So we get the checksums
// for both of those, and merge them together to make the final checksum.
//
// We also want the checksum to change any time our serialization format changes. If
// the format has changed, all previous versions should be invalidated.
var projectChecksumState = await document.Project.State.GetStateChecksumsAsync(cancellationToken).ConfigureAwait(false);
var parseOptionsChecksum = projectChecksumState.ParseOptions;
var documentChecksumState = await document.State.GetStateChecksumsAsync(cancellationToken).ConfigureAwait(false);
var textChecksum = documentChecksumState.Text;
return Checksum.Create(WellKnownSynchronizationKind.SyntaxTreeIndex, new[] { textChecksum, parseOptionsChecksum });
return Checksum.Create(
WellKnownSynchronizationKind.SyntaxTreeIndex,
new[] { textChecksum, parseOptionsChecksum, SerializationFormatChecksum });
}
private async Task<bool> SaveAsync(
Document document, CancellationToken cancellationToken)
{
var solution = document.Project.Solution;
var persistentStorageService = (IPersistentStorageService2)solution.Workspace.Services.GetService<IPersistentStorageService>();
var persistentStorageService = (IChecksummedPersistentStorageService)solution.Workspace.Services.GetService<IPersistentStorageService>();
try
{
......@@ -101,11 +79,10 @@ private void WriteFormatAndChecksum(ObjectWriter writer, string formatVersion)
using (var stream = SerializableBytes.CreateWritableStream())
using (var writer = new ObjectWriter(stream, cancellationToken: cancellationToken))
{
this.WriteFormatAndChecksum(writer, SerializationFormat);
this.WriteTo(writer);
stream.Position = 0;
return await storage.WriteStreamAsync(document, PersistenceName, stream, cancellationToken).ConfigureAwait(false);
return await storage.WriteStreamAsync(document, PersistenceName, stream, this.Checksum, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception e) when (IOUtilities.IsNormalIOException(e))
......@@ -120,19 +97,19 @@ private void WriteFormatAndChecksum(ObjectWriter writer, string formatVersion)
Document document, Checksum checksum, CancellationToken cancellationToken)
{
var solution = document.Project.Solution;
var persistentStorageService = (IPersistentStorageService2)solution.Workspace.Services.GetService<IPersistentStorageService>();
var persistentStorageService = (IChecksummedPersistentStorageService)solution.Workspace.Services.GetService<IPersistentStorageService>();
// check whether we already have info for this document
try
{
using (var storage = persistentStorageService.GetStorage(solution, checkBranchId: false))
using (var stream = await storage.ReadStreamAsync(document, PersistenceName, cancellationToken).ConfigureAwait(false))
using (var reader = ObjectReader.TryGetReader(stream))
{
if (reader != null)
{
return FormatAndChecksumMatches(reader, SerializationFormat, checksum);
}
// Check if we've already stored a checksum and it matches the checksum we
// expect. If so, we're already precalculated and don't have to recompute
// this index. Otherwise if we don't have a checksum, or the checksums don't
// match, go ahead and recompute it.
var persistedChecksum = await storage.ReadChecksumAsync(document, PersistenceName, cancellationToken).ConfigureAwait(false);
return persistedChecksum == checksum;
}
}
catch (Exception e) when (IOUtilities.IsNormalIOException(e))
......
src/Workspaces/Core/Portable/Storage/AbstractPersistentStorageService.cs
浏览文件 @ 3f487e01
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
......@@ -18,7 +17,7 @@ namespace Microsoft.CodeAnalysis.Storage
/// A service that enables storing and retrieving of information associated with solutions,
/// projects or documents across runtime sessions.
/// </summary>
internal abstract partial class AbstractPersistentStorageService : IPersistentStorageService2
internal abstract partial class AbstractPersistentStorageService : IChecksummedPersistentStorageService
{
private readonly IOptionService _optionService;
private readonly IPersistentStorageLocationService _locationService;
......@@ -28,7 +27,7 @@ internal abstract partial class AbstractPersistentStorageService : IPersistentSt
/// This lock guards all mutable fields in this type.
/// </summary>
private readonly object _lock = new object();
private ReferenceCountedDisposable<IPersistentStorage> _currentPersistentStorage;
private ReferenceCountedDisposable<IChecksummedPersistentStorage> _currentPersistentStorage;
private SolutionId _currentPersistentStorageSolutionId;
private bool _subscribedToLocationServiceChangeEvents;
......@@ -43,13 +42,19 @@ internal abstract partial class AbstractPersistentStorageService : IPersistentSt
}
protected abstract string GetDatabaseFilePath(string workingFolderPath);
protected abstract bool TryOpenDatabase(Solution solution, string workingFolderPath, string databaseFilePath, out IPersistentStorage storage);
protected abstract bool TryOpenDatabase(Solution solution, string workingFolderPath, string databaseFilePath, out IChecksummedPersistentStorage storage);
protected abstract bool ShouldDeleteDatabase(Exception exception);
public IPersistentStorage GetStorage(Solution solution)
IPersistentStorage IPersistentStorageService.GetStorage(Solution solution)
=> this.GetStorage(solution);
IPersistentStorage IPersistentStorageService2.GetStorage(Solution solution, bool checkBranchId)
=> this.GetStorage(solution, checkBranchId);
public IChecksummedPersistentStorage GetStorage(Solution solution)
=> GetStorage(solution, checkBranchId: true);
public IPersistentStorage GetStorage(Solution solution, bool checkBranchId)
public IChecksummedPersistentStorage GetStorage(Solution solution, bool checkBranchId)
{
if (!DatabaseSupported(solution, checkBranchId))
{
......@@ -143,7 +148,7 @@ private bool SolutionSizeAboveThreshold(Solution solution)
return size >= threshold;
}
private ReferenceCountedDisposable<IPersistentStorage> TryCreatePersistentStorage(Solution solution, string workingFolderPath)
private ReferenceCountedDisposable<IChecksummedPersistentStorage> TryCreatePersistentStorage(Solution solution, string workingFolderPath)
{
// Attempt to create the database up to two times. The first time we may encounter
// some sort of issue (like DB corruption). We'll then try to delete the DB and can
......@@ -152,7 +157,7 @@ private ReferenceCountedDisposable<IPersistentStorage> TryCreatePersistentStorag
if (TryCreatePersistentStorage(solution, workingFolderPath, out var persistentStorage) ||
TryCreatePersistentStorage(solution, workingFolderPath, out persistentStorage))
{
return new ReferenceCountedDisposable<IPersistentStorage>(persistentStorage);
return new ReferenceCountedDisposable<IChecksummedPersistentStorage>(persistentStorage);
}
// okay, can't recover, then use no op persistent service
......@@ -163,7 +168,7 @@ private ReferenceCountedDisposable<IPersistentStorage> TryCreatePersistentStorag
private bool TryCreatePersistentStorage(
Solution solution,
string workingFolderPath,
out IPersistentStorage persistentStorage)
out IChecksummedPersistentStorage persistentStorage)
{
persistentStorage = null;
......@@ -195,7 +200,7 @@ private ReferenceCountedDisposable<IPersistentStorage> TryCreatePersistentStorag
private void LocationServiceStorageLocationChanging(object sender, PersistentStorageLocationChangingEventArgs e)
{
ReferenceCountedDisposable<IPersistentStorage> storage = null;
ReferenceCountedDisposable<IChecksummedPersistentStorage> storage = null;
lock (_lock)
{
......@@ -230,16 +235,16 @@ private void LocationServiceStorageLocationChanging(object sender, PersistentSto
/// A trivial wrapper that we can hand out for instances from the <see cref="AbstractPersistentStorageService"/>
/// that wraps the underlying <see cref="IPersistentStorage"/> singleton.
/// </summary>
private sealed class PersistentStorageReferenceCountedDisposableWrapper : IPersistentStorage
private sealed class PersistentStorageReferenceCountedDisposableWrapper : IChecksummedPersistentStorage
{
private readonly ReferenceCountedDisposable<IPersistentStorage> _storage;
private readonly ReferenceCountedDisposable<IChecksummedPersistentStorage> _storage;
private PersistentStorageReferenceCountedDisposableWrapper(ReferenceCountedDisposable<IPersistentStorage> storage)
private PersistentStorageReferenceCountedDisposableWrapper(ReferenceCountedDisposable<IChecksummedPersistentStorage> storage)
{
_storage = storage;
}
public static IPersistentStorage AddReferenceCountToAndCreateWrapper(ReferenceCountedDisposable<IPersistentStorage> storage)
public static IChecksummedPersistentStorage AddReferenceCountToAndCreateWrapper(ReferenceCountedDisposable<IChecksummedPersistentStorage> storage)
{
return new PersistentStorageReferenceCountedDisposableWrapper(storage.TryAddReference());
}
......@@ -249,23 +254,50 @@ public void Dispose()
_storage.Dispose();
}
public Task<Stream> ReadStreamAsync(string name, CancellationToken cancellationToken = default)
public Task<Checksum> ReadChecksumAsync(string name, CancellationToken cancellationToken)
=> _storage.Target.ReadChecksumAsync(name, cancellationToken);
public Task<Checksum> ReadChecksumAsync(Project project, string name, CancellationToken cancellationToken)
=> _storage.Target.ReadChecksumAsync(project, name, cancellationToken);
public Task<Checksum> ReadChecksumAsync(Document document, string name, CancellationToken cancellationToken)
=> _storage.Target.ReadChecksumAsync(document, name, cancellationToken);
public Task<Stream> ReadStreamAsync(string name, CancellationToken cancellationToken)
=> _storage.Target.ReadStreamAsync(name, cancellationToken);
public Task<Stream> ReadStreamAsync(Project project, string name, CancellationToken cancellationToken = default)
public Task<Stream> ReadStreamAsync(Project project, string name, CancellationToken cancellationToken)
=> _storage.Target.ReadStreamAsync(project, name, cancellationToken);
public Task<Stream> ReadStreamAsync(Document document, string name, CancellationToken cancellationToken = default)
public Task<Stream> ReadStreamAsync(Document document, string name, CancellationToken cancellationToken)
=> _storage.Target.ReadStreamAsync(document, name, cancellationToken);
public Task<bool> WriteStreamAsync(string name, Stream stream, CancellationToken cancellationToken = default)
public Task<Stream> ReadStreamAsync(string name, Checksum checksum, CancellationToken cancellationToken)
=> _storage.Target.ReadStreamAsync(name, checksum, cancellationToken);
public Task<Stream> ReadStreamAsync(Project project, string name, Checksum checksum, CancellationToken cancellationToken)
=> _storage.Target.ReadStreamAsync(project, name, checksum, cancellationToken);
public Task<Stream> ReadStreamAsync(Document document, string name, Checksum checksum, CancellationToken cancellationToken)
=> _storage.Target.ReadStreamAsync(document, name, checksum, cancellationToken);
public Task<bool> WriteStreamAsync(string name, Stream stream, CancellationToken cancellationToken)
=> _storage.Target.WriteStreamAsync(name, stream, cancellationToken);
public Task<bool> WriteStreamAsync(Project project, string name, Stream stream, CancellationToken cancellationToken = default)
public Task<bool> WriteStreamAsync(Project project, string name, Stream stream, CancellationToken cancellationToken)
=> _storage.Target.WriteStreamAsync(project, name, stream, cancellationToken);
public Task<bool> WriteStreamAsync(Document document, string name, Stream stream, CancellationToken cancellationToken = default)
public Task<bool> WriteStreamAsync(Document document, string name, Stream stream, CancellationToken cancellationToken)
=> _storage.Target.WriteStreamAsync(document, name, stream, cancellationToken);
public Task<bool> WriteStreamAsync(string name, Stream stream, Checksum checksum, CancellationToken cancellationToken)
=> _storage.Target.WriteStreamAsync(name, stream, checksum, cancellationToken);
public Task<bool> WriteStreamAsync(Project project, string name, Stream stream, Checksum checksum, CancellationToken cancellationToken)
=> _storage.Target.WriteStreamAsync(project, name, stream, checksum, cancellationToken);
public Task<bool> WriteStreamAsync(Document document, string name, Stream stream, Checksum checksum, CancellationToken cancellationToken)
=> _storage.Target.WriteStreamAsync(document, name, stream, checksum, cancellationToken);
}
}
}
src/Workspaces/Core/Portable/Storage/PersistenceStorageServiceFactory.cs
浏览文件 @ 3f487e01
......@@ -12,7 +12,6 @@ namespace Microsoft.CodeAnalysis.Storage
[ExportWorkspaceServiceFactory(typeof(IPersistentStorageService), ServiceLayer.Desktop), Shared]
internal class PersistenceStorageServiceFactory : IWorkspaceServiceFactory
{
private readonly object _gate = new object();
private readonly ISolutionSizeTracker _solutionSizeTracker;
[ImportingConstructor]
......
src/Workspaces/Core/Portable/Storage/SQLite/Interop/SqlConnection.cs
浏览文件 @ 3f487e01
......@@ -2,6 +2,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using Microsoft.CodeAnalysis.ErrorReporting;
using Microsoft.CodeAnalysis.Host;
......@@ -192,13 +193,13 @@ public void RunInTransaction<TState>(Action<TState> action, TState state)
}
private void Rollback(bool throwOnError)
=> this.ExecuteCommand("rollback transaction", throwOnError);
=> ExecuteCommand("rollback transaction", throwOnError);
public int LastInsertRowId()
=> (int)raw.sqlite3_last_insert_rowid(_handle);
[PerformanceSensitive("https://github.com/dotnet/roslyn/issues/36114", AllowCaptures = false)]
public Stream ReadBlob(string dataTableName, string dataColumnName, long rowId)
public Stream ReadBlob_MustRunInTransaction(string tableName, string columnName, long rowId)
{
// NOTE: we do need to do the blob reading in a transaction because of the
// following: https://www.sqlite.org/c3ref/blob_open.html
......@@ -209,15 +210,11 @@ public Stream ReadBlob(string dataTableName, string dataColumnName, long rowId)
// the one the BLOB handle is open on. Calls to sqlite3_blob_read() and
// sqlite3_blob_write() for an expired BLOB handle fail with a return code of
// SQLITE_ABORT.
var stream = RunInTransaction(
state => state.self.ReadBlob_InTransaction(state.dataTableName, state.dataColumnName, state.rowId),
(self: this, dataTableName, dataColumnName, rowId));
return stream;
}
if (!IsInTransaction)
{
throw new InvalidOperationException("Must read blobs within a transaction to prevent corruption!");
}
private Stream ReadBlob_InTransaction(string tableName, string columnName, long rowId)
{
const int ReadOnlyFlags = 0;
var result = raw.sqlite3_blob_open(_handle, "main", tableName, columnName, rowId, ReadOnlyFlags, out var blob);
if (result == raw.SQLITE_ERROR)
......
src/Workspaces/Core/Portable/Storage/SQLite/SQLitePersistentStorage.Accessor.cs
浏览文件 @ 3f487e01
......@@ -24,6 +24,8 @@ internal partial class SQLitePersistentStorage
private abstract class Accessor<TKey, TWriteQueueKey, TDatabaseId>
{
protected readonly SQLitePersistentStorage Storage;
private readonly string _select_rowid_from_0_where_1;
private readonly string _insert_or_replace_into_0_1_2_3_value;
/// <summary>
/// Queue of actions we want to perform all at once against the DB in a single transaction.
......@@ -42,6 +44,8 @@ private abstract class Accessor<TKey, TWriteQueueKey, TDatabaseId>
public Accessor(SQLitePersistentStorage storage)
{
Storage = storage;
_select_rowid_from_0_where_1 = $@"select rowid from ""{DataTableName}"" where ""{DataIdColumnName}"" = ?";
_insert_or_replace_into_0_1_2_3_value = $@"insert or replace into ""{DataTableName}""(""{DataIdColumnName}"",""{ChecksumColumnName}"",""{DataColumnName}"") values (?,?,?)";
}
protected abstract string DataTableName { get; }
......@@ -50,12 +54,26 @@ public Accessor(SQLitePersistentStorage storage)
protected abstract void BindFirstParameter(SqlStatement statement, TDatabaseId dataId);
protected abstract TWriteQueueKey GetWriteQueueKey(TKey key);
public async Task<Stream> ReadStreamAsync(TKey key, CancellationToken cancellationToken)
public async Task<Checksum> ReadChecksumAsync(TKey key, CancellationToken cancellationToken)
{
// Note: we're technically fully synchronous. However, we're called from several
// async methods. We just return a Task<stream> here so that all our callers don't
// need to call Task.FromResult on us.
using (var stream = await ReadBlobColumnAsync(key, ChecksumColumnName, checksumOpt: null, cancellationToken).ConfigureAwait(false))
using (var reader = ObjectReader.TryGetReader(stream, cancellationToken))
{
if (reader != null)
{
return Checksum.ReadFrom(reader);
}
}
return null;
}
public Task<Stream> ReadStreamAsync(TKey key, Checksum checksum, CancellationToken cancellationToken)
=> ReadBlobColumnAsync(key, DataColumnName, checksum, cancellationToken);
private async Task<Stream> ReadBlobColumnAsync(
TKey key, string columnName, Checksum checksumOpt, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (!Storage._shutdownTokenSource.IsCancellationRequested)
......@@ -78,7 +96,9 @@ public async Task<Stream> ReadStreamAsync(TKey key, CancellationToken cancellati
using (var pooledConnection = Storage.GetPooledConnection())
{
// Lookup the row from the DocumentData table corresponding to our dataId.
return ReadBlob(pooledConnection.Connection, dataId);
return ReadBlob(
pooledConnection.Connection, dataId, columnName,
checksumOpt, cancellationToken);
}
}
catch (Exception ex)
......@@ -92,7 +112,7 @@ public async Task<Stream> ReadStreamAsync(TKey key, CancellationToken cancellati
}
public async Task<bool> WriteStreamAsync(
TKey key, Stream stream, CancellationToken cancellationToken)
TKey key, Stream stream, Checksum checksumOpt, CancellationToken cancellationToken)
{
// Note: we're technically fully synchronous. However, we're called from several
// async methods. We just return a Task<bool> here so that all our callers don't
......@@ -112,14 +132,22 @@ public async Task<Stream> ReadStreamAsync(TKey key, CancellationToken cancellati
if (haveDataId)
{
var (bytes, length, pooled) = GetBytes(stream);
var (checksumBytes, checksumLength, checksumPooled) = GetBytes(checksumOpt, cancellationToken);
var (dataBytes, dataLength, dataPooled) = GetBytes(stream);
await AddWriteTaskAsync(key, con =>
{
InsertOrReplaceBlob(con, dataId, bytes, length);
if (pooled)
InsertOrReplaceBlob(con, dataId,
checksumBytes, checksumLength,
dataBytes, dataLength);
if (dataPooled)
{
ReturnPooledBytes(bytes);
ReturnPooledBytes(dataBytes);
}
if (checksumPooled)
{
ReturnPooledBytes(checksumBytes);
}
}, cancellationToken).ConfigureAwait(false);
......@@ -136,22 +164,52 @@ private Task FlushPendingWritesAsync(TKey key, CancellationToken cancellationTok
private Task AddWriteTaskAsync(TKey key, Action<SqlConnection> action, CancellationToken cancellationToken)
=> Storage.AddWriteTaskAsync(_writeQueueKeyToWrites, GetWriteQueueKey(key), action, cancellationToken);
private Stream ReadBlob(SqlConnection connection, TDatabaseId dataId)
private Stream ReadBlob(
SqlConnection connection, TDatabaseId dataId, string columnName,
Checksum checksumOpt, CancellationToken cancellationToken)
{
// Note: it's possible that someone may write to this row between when we
// get the row ID above and now. That's fine. We'll just read the new
// bytes that have been written to this location. Note that only the
// data for a row in our system can change, the ID will always stay the
// same, and the data will always be valid for our ID. So there is no
// safety issue here.
if (TryGetRowId(connection, dataId, out var rowId))
{
// Note: it's possible that someone may write to this row between when we
// get the row ID above and now. That's fine. We'll just read the new
// bytes that have been written to this location. Note that only the
// data for a row in our system can change, the ID will always stay the
// same, and the data will always be valid for our ID. So there is no
// safety issue here.
return connection.ReadBlob(DataTableName, DataColumnName, rowId);
// Have to run the blob reading in a transaction. This is necessary
// for two reasons. First, blob reading outside a transaction is not
// safe to do with the sqlite API. It may produce corrupt bits if
// another thread is writing to the blob. Second, if a checksum was
// passed in, we need to validate that the checksums match. This is
// only safe if we are in a transaction and no-one else can race with
// us.
return connection.RunInTransaction((tuple) =>
{
// If we were passed a checksum, make sure it matches what we have
// stored in the table already. If they don't match, don't read
// out the data value at all.
if (tuple.checksumOpt != null &&
!ChecksumsMatch_MustRunInTransaction(tuple.connection, tuple.rowId, tuple.checksumOpt, cancellationToken))
{
return null;
}
return connection.ReadBlob_MustRunInTransaction(tuple.self.DataTableName, tuple.columnName, tuple.rowId);
}, (self: this, connection, columnName, checksumOpt, rowId));
}
return null;
}
private bool ChecksumsMatch_MustRunInTransaction(SqlConnection connection, long rowId, Checksum checksum, CancellationToken cancellationToken)
{
using (var checksumStream = connection.ReadBlob_MustRunInTransaction(DataTableName, ChecksumColumnName, rowId))
using (var reader = ObjectReader.TryGetReader(checksumStream, cancellationToken))
{
return reader != null && Checksum.ReadFrom(reader) == checksum;
}
}
protected bool GetAndVerifyRowId(SqlConnection connection, long dataId, out long rowId)
{
// For the Document and Project tables, our dataId is our rowId:
......@@ -191,8 +249,7 @@ private bool TryGetRowIdWorker(SqlConnection connection, TDatabaseId dataId, out
// ROWID, _ROWID_, or OID. Except if you declare an ordinary table column to use one
// of those special names, then the use of that name will refer to the declared column
// not to the internal ROWID.
using (var resettableStatement = connection.GetResettableStatement(
$@"select rowid from ""{this.DataTableName}"" where ""{DataIdColumnName}"" = ?"))
using (var resettableStatement = connection.GetResettableStatement(_select_rowid_from_0_where_1))
{
var statement = resettableStatement.Statement;
......@@ -212,16 +269,18 @@ private bool TryGetRowIdWorker(SqlConnection connection, TDatabaseId dataId, out
}
private void InsertOrReplaceBlob(
SqlConnection conection, TDatabaseId dataId, byte[] bytes, int length)
SqlConnection connection, TDatabaseId dataId,
byte[] checksumBytes, int checksumLength,
byte[] dataBytes, int dataLength)
{
using (var resettableStatement = conection.GetResettableStatement(
$@"insert or replace into ""{this.DataTableName}""(""{DataIdColumnName}"",""{DataColumnName}"") values (?,?)"))
using (var resettableStatement = connection.GetResettableStatement(_insert_or_replace_into_0_1_2_3_value))
{
var statement = resettableStatement.Statement;
// Binding indices are 1 based.
BindFirstParameter(statement, dataId);
statement.BindBlobParameter(parameterIndex: 2, value: bytes, length: length);
statement.BindBlobParameter(parameterIndex: 2, value: checksumBytes, length: checksumLength);
statement.BindBlobParameter(parameterIndex: 3, value: dataBytes, length: dataLength);
statement.Step();
}
......
src/Workspaces/Core/Portable/Storage/SQLite/SQLitePersistentStorage.cs
浏览文件 @ 3f487e01
......@@ -16,7 +16,12 @@ namespace Microsoft.CodeAnalysis.SQLite
/// </summary>
internal partial class SQLitePersistentStorage : AbstractPersistentStorage
{
private const string Version = "1";
// Version history.
// 1. Initial use of sqlite as the persistence layer. Simple key->value storage tables.
// 2. Updated to store checksums. Tables now key->(checksum,value). Allows for reading
// and validating checksums without the overhead of reading the full 'value' into
// memory.
private const string Version = "2";
/// <summary>
/// Inside the DB we have a table dedicated to storing strings that also provides a unique
......@@ -49,16 +54,16 @@ internal partial class SQLitePersistentStorage : AbstractPersistentStorage
/// The format of the table is:
///
/// SolutionData
/// -----------------------------------------------
/// | DataId (primary key, varchar) | Data (blob) |
/// -----------------------------------------------
/// -------------------------------------------------------------------
/// | DataId (primary key, varchar) | | Checksum (blob) | Data (blob) |
/// -------------------------------------------------------------------
/// </summary>
private const string SolutionDataTableName = "SolutionData" + Version;
/// <summary>
/// Inside the DB we have a table for data that we want associated with a <see cref="Project"/>.
/// The data is keyed off of an integral value produced by combining the ID of the Project and
/// the ID of the name of the data (see <see cref="SQLitePersistentStorage.ReadStreamAsync(Project, string, CancellationToken)"/>.
/// the ID of the name of the data (see <see cref="SQLitePersistentStorage.ReadStreamAsync(Project, string, Checksum, CancellationToken)"/>.
///
/// This gives a very efficient integral key, and means that the we only have to store a
/// single mapping from stream name to ID in the string table.
......@@ -66,16 +71,16 @@ internal partial class SQLitePersistentStorage : AbstractPersistentStorage
/// The format of the table is:
///
/// ProjectData
/// -----------------------------------------------
/// | DataId (primary key, integer) | Data (blob) |
/// -----------------------------------------------
/// -------------------------------------------------------------------
/// | DataId (primary key, integer) | | Checksum (blob) | Data (blob) |
/// -------------------------------------------------------------------
/// </summary>
private const string ProjectDataTableName = "ProjectData" + Version;
/// <summary>
/// Inside the DB we have a table for data that we want associated with a <see cref="Document"/>.
/// The data is keyed off of an integral value produced by combining the ID of the Document and
/// the ID of the name of the data (see <see cref="SQLitePersistentStorage.ReadStreamAsync(Document, string, CancellationToken)"/>.
/// the ID of the name of the data (see <see cref="SQLitePersistentStorage.ReadStreamAsync(Document, string, Checksum, CancellationToken)"/>.
///
/// This gives a very efficient integral key, and means that the we only have to store a
/// single mapping from stream name to ID in the string table.
......@@ -83,13 +88,14 @@ internal partial class SQLitePersistentStorage : AbstractPersistentStorage
/// The format of the table is:
///
/// DocumentData
/// ----------------------------------------------
/// | DataId (primary key, integer) | Data (blob) |
/// ----------------------------------------------
/// -------------------------------------------------------------------
/// | DataId (primary key, integer) | | Checksum (blob) | Data (blob) |
/// -------------------------------------------------------------------
/// </summary>
private const string DocumentDataTableName = "DocumentData" + Version;
private const string DataIdColumnName = "DataId";
private const string ChecksumColumnName = "Checksum";
private const string DataColumnName = "Data";
private readonly CancellationTokenSource _shutdownTokenSource = new CancellationTokenSource();
......@@ -106,6 +112,12 @@ internal partial class SQLitePersistentStorage : AbstractPersistentStorage
private readonly ProjectAccessor _projectAccessor;
private readonly DocumentAccessor _documentAccessor;
// cached query strings
private readonly string _select_star_from_0;
private readonly string _insert_into_0_1_values;
private readonly string _select_star_from_0_where_1_limit_one;
// We pool connections to the DB so that we don't have to take the hit of
// reconnecting. The connections also cache the prepared statements used
// to get/set data from the db. A connection is safe to use by one thread
......@@ -127,6 +139,10 @@ internal partial class SQLitePersistentStorage : AbstractPersistentStorage
_solutionAccessor = new SolutionAccessor(this);
_projectAccessor = new ProjectAccessor(this);
_documentAccessor = new DocumentAccessor(this);
_select_star_from_0 = $@"select * from ""{StringInfoTableName}""";
_insert_into_0_1_values = $@"insert into ""{StringInfoTableName}""(""{DataColumnName}"") values (?)";
_select_star_from_0_where_1_limit_one = $@"select * from ""{StringInfoTableName}"" where (""{DataColumnName}"" = ?) limit 1";
}
private SqlConnection GetConnection()
......@@ -141,7 +157,7 @@ private SqlConnection GetConnection()
}
// Otherwise create a new connection.
return SqlConnection.Create(_faultInjectorOpt, this.DatabaseFile);
return SqlConnection.Create(_faultInjectorOpt, DatabaseFile);
}
private void ReleaseConnection(SqlConnection connection)
......@@ -248,16 +264,19 @@ public void Initialize(Solution solution)
connection.ExecuteCommand(
$@"create table if not exists ""{SolutionDataTableName}"" (
""{DataIdColumnName}"" varchar primary key not null,
""{ChecksumColumnName}"" blob,
""{DataColumnName}"" blob)");
connection.ExecuteCommand(
$@"create table if not exists ""{ProjectDataTableName}"" (
""{DataIdColumnName}"" integer primary key not null,
""{ChecksumColumnName}"" blob,
""{DataColumnName}"" blob)");
connection.ExecuteCommand(
$@"create table if not exists ""{DocumentDataTableName}"" (
""{DataIdColumnName}"" integer primary key not null,
""{ChecksumColumnName}"" blob,
""{DataColumnName}"" blob)");
// Also get the known set of string-to-id mappings we already have in the DB.
......
src/Workspaces/Core/Portable/Storage/SQLite/SQLitePersistentStorageService.cs
浏览文件 @ 3f487e01
......@@ -86,7 +86,7 @@ protected override string GetDatabaseFilePath(string workingFolderPath)
}
protected override bool TryOpenDatabase(
Solution solution, string workingFolderPath, string databaseFilePath, out IPersistentStorage storage)
Solution solution, string workingFolderPath, string databaseFilePath, out IChecksummedPersistentStorage storage)
{
if (!TryInitializeLibraries())
{
......@@ -104,7 +104,6 @@ protected override string GetDatabaseFilePath(string workingFolderPath)
}
SQLitePersistentStorage sqlStorage = null;
try
{
sqlStorage = new SQLitePersistentStorage(
......
src/Workspaces/Core/Portable/Storage/SQLite/SQLitePersistentStorage_DocumentSerialization.cs
浏览文件 @ 3f487e01
......@@ -9,11 +9,14 @@ namespace Microsoft.CodeAnalysis.SQLite
{
internal partial class SQLitePersistentStorage
{
public override Task<Stream> ReadStreamAsync(Document document, string name, CancellationToken cancellationToken = default)
=> _documentAccessor.ReadStreamAsync((document, name), cancellationToken);
public override Task<Checksum> ReadChecksumAsync(Document document, string name, CancellationToken cancellationToken)
=> _documentAccessor.ReadChecksumAsync((document, name), cancellationToken);
public override Task<bool> WriteStreamAsync(Document document, string name, Stream stream, CancellationToken cancellationToken = default)
=> _documentAccessor.WriteStreamAsync((document, name), stream, cancellationToken);
public override Task<Stream> ReadStreamAsync(Document document, string name, Checksum checksum, CancellationToken cancellationToken = default)
=> _documentAccessor.ReadStreamAsync((document, name), checksum, cancellationToken);
public override Task<bool> WriteStreamAsync(Document document, string name, Stream stream, Checksum checksum, CancellationToken cancellationToken = default)
=> _documentAccessor.WriteStreamAsync((document, name), stream, checksum, cancellationToken);
/// <summary>
/// <see cref="Accessor{TKey, TWriteQueueKey, TDatabaseId}"/> responsible for storing and
......
src/Workspaces/Core/Portable/Storage/SQLite/SQLitePersistentStorage_Helpers.cs
浏览文件 @ 3f487e01
......@@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using Roslyn.Utilities;
using static System.FormattableString;
......@@ -19,6 +20,28 @@ private static string GetDocumentIdString(int projectId, int documentPathId, int
private static long CombineInt32ValuesToInt64(int v1, int v2)
=> ((long)v1 << 32) | (long)v2;
private static (byte[] bytes, int length, bool fromPool) GetBytes(
Checksum checksumOpt, CancellationToken cancellationToken)
{
// If we weren't passed a checsum, just pass the singleton empty byte array.
// Note: we don't add this to/from our pool. But it likely woudn't be a problem
// for us to do that as this instance can't actually be mutated since it's just
// an empty array.
if (checksumOpt == null)
{
return (Array.Empty<byte>(), length: 0, fromPool: false);
}
using (var stream = SerializableBytes.CreateWritableStream())
using (var writer = new ObjectWriter(stream, cancellationToken: cancellationToken))
{
checksumOpt.WriteTo(writer);
stream.Position = 0;
return GetBytes(stream);
}
}
private static (byte[] bytes, int length, bool fromPool) GetBytes(Stream stream)
{
// Attempt to copy into a pooled byte[] if the stream length is known and it's
......
src/Workspaces/Core/Portable/Storage/SQLite/SQLitePersistentStorage_ProjectSerialization.cs
浏览文件 @ 3f487e01
......@@ -9,11 +9,14 @@ namespace Microsoft.CodeAnalysis.SQLite
{
internal partial class SQLitePersistentStorage
{
public override Task<Stream> ReadStreamAsync(Project project, string name, CancellationToken cancellationToken = default)
=> _projectAccessor.ReadStreamAsync((project, name), cancellationToken);
public override Task<Checksum> ReadChecksumAsync(Project project, string name, CancellationToken cancellationToken)
=> _projectAccessor.ReadChecksumAsync((project, name), cancellationToken);
public override Task<bool> WriteStreamAsync(Project project, string name, Stream stream, CancellationToken cancellationToken = default)
=> _projectAccessor.WriteStreamAsync((project, name), stream, cancellationToken);
public override Task<Stream> ReadStreamAsync(Project project, string name, Checksum checksum, CancellationToken cancellationToken = default)
=> _projectAccessor.ReadStreamAsync((project, name), checksum, cancellationToken);
public override Task<bool> WriteStreamAsync(Project project, string name, Stream stream, Checksum checksum, CancellationToken cancellationToken = default)
=> _projectAccessor.WriteStreamAsync((project, name), stream, checksum, cancellationToken);
/// <summary>
/// <see cref="Accessor{TKey, TWriteQueueKey, TDatabaseId}"/> responsible for storing and
......
src/Workspaces/Core/Portable/Storage/SQLite/SQLitePersistentStorage_SolutionSerialization.cs
浏览文件 @ 3f487e01
......@@ -9,11 +9,14 @@ namespace Microsoft.CodeAnalysis.SQLite
{
internal partial class SQLitePersistentStorage
{
public override Task<Stream> ReadStreamAsync(string name, CancellationToken cancellationToken)
=> _solutionAccessor.ReadStreamAsync(name, cancellationToken);
public override Task<Checksum> ReadChecksumAsync(string name, CancellationToken cancellationToken)
=> _solutionAccessor.ReadChecksumAsync(name, cancellationToken);
public override Task<bool> WriteStreamAsync(string name, Stream stream, CancellationToken cancellationToken)
=> _solutionAccessor.WriteStreamAsync(name, stream, cancellationToken);
public override Task<Stream> ReadStreamAsync(string name, Checksum checksum, CancellationToken cancellationToken)
=> _solutionAccessor.ReadStreamAsync(name, checksum, cancellationToken);
public override Task<bool> WriteStreamAsync(string name, Stream stream, Checksum checksum, CancellationToken cancellationToken)
=> _solutionAccessor.WriteStreamAsync(name, stream, checksum, cancellationToken);
/// <summary>
/// <see cref="Accessor{TKey, TWriteQueueKey, TDatabaseId}"/> responsible for storing and
......
src/Workspaces/Core/Portable/Storage/SQLite/SQLitePersistentStorage_StringIds.cs
浏览文件 @ 3f487e01
......@@ -16,8 +16,7 @@ private bool TryFetchStringTable(SqlConnection connection)
{
try
{
using (var resettableStatement = connection.GetResettableStatement(
$@"select * from ""{StringInfoTableName}"""))
using (var resettableStatement = connection.GetResettableStatement(_select_star_from_0))
{
var statement = resettableStatement.Statement;
while (statement.Step() == Result.ROW)
......@@ -55,7 +54,7 @@ private bool TryFetchStringTable(SqlConnection connection)
// First see if we've cached the ID for this value locally. If so, just return
// what we already have.
if (_stringToIdMap.TryGetValue(value, out int existingId))
if (_stringToIdMap.TryGetValue(value, out var existingId))
{
return existingId;
}
......@@ -109,17 +108,16 @@ private bool TryFetchStringTable(SqlConnection connection)
return null;
}
private static int InsertStringIntoDatabase_MustRunInTransaction(SqlConnection connection, string value)
private int InsertStringIntoDatabase_MustRunInTransaction(SqlConnection connection, string value)
{
if (!connection.IsInTransaction)
{
throw new InvalidOperationException("Must call this while connection has transaction open");
}
int id = -1;
var id = -1;
using (var resettableStatement = connection.GetResettableStatement(
$@"insert into ""{StringInfoTableName}""(""{DataColumnName}"") values (?)"))
using (var resettableStatement = connection.GetResettableStatement(_insert_into_0_1_values))
{
var statement = resettableStatement.Statement;
......@@ -145,8 +143,7 @@ private static int InsertStringIntoDatabase_MustRunInTransaction(SqlConnection c
{
try
{
using (var resettableStatement = connection.GetResettableStatement(
$@"select * from ""{StringInfoTableName}"" where (""{DataColumnName}"" = ?) limit 1"))
using (var resettableStatement = connection.GetResettableStatement(_select_star_from_0_where_1_limit_one))
{
var statement = resettableStatement.Statement;
......
src/Workspaces/Core/Portable/Workspace/Host/PersistentStorage/AbstractPersistentStorage.cs
浏览文件 @ 3f487e01
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.Options;
using Roslyn.Utilities;
namespace Microsoft.CodeAnalysis.Host
{
internal abstract class AbstractPersistentStorage : IPersistentStorage
internal abstract class AbstractPersistentStorage : IChecksummedPersistentStorage
{
public string WorkingFolderPath { get; }
public string SolutionFilePath { get; }
......@@ -32,13 +29,36 @@ internal abstract class AbstractPersistentStorage : IPersistentStorage
}
}
public abstract Task<Stream> ReadStreamAsync(string name, CancellationToken cancellationToken = default);
public abstract Task<Stream> ReadStreamAsync(Project project, string name, CancellationToken cancellationToken = default);
public abstract Task<Stream> ReadStreamAsync(Document document, string name, CancellationToken cancellationToken = default);
public abstract Task<bool> WriteStreamAsync(string name, Stream stream, CancellationToken cancellationToken = default);
public abstract Task<bool> WriteStreamAsync(Project project, string name, Stream stream, CancellationToken cancellationToken = default);
public abstract Task<bool> WriteStreamAsync(Document document, string name, Stream stream, CancellationToken cancellationToken = default);
public abstract void Dispose();
public abstract Task<Checksum> ReadChecksumAsync(string name, CancellationToken cancellationToken);
public abstract Task<Checksum> ReadChecksumAsync(Project project, string name, CancellationToken cancellationToken);
public abstract Task<Checksum> ReadChecksumAsync(Document document, string name, CancellationToken cancellationToken);
public abstract Task<Stream> ReadStreamAsync(string name, Checksum checksum, CancellationToken cancellationToken);
public abstract Task<Stream> ReadStreamAsync(Project project, string name, Checksum checksum, CancellationToken cancellationToken);
public abstract Task<Stream> ReadStreamAsync(Document document, string name, Checksum checksum, CancellationToken cancellationToken);
public abstract Task<bool> WriteStreamAsync(string name, Stream stream, Checksum checksum, CancellationToken cancellationToken);
public abstract Task<bool> WriteStreamAsync(Project project, string name, Stream stream, Checksum checksum, CancellationToken cancellationToken);
public abstract Task<bool> WriteStreamAsync(Document document, string name, Stream stream, Checksum checksum, CancellationToken cancellationToken);
public Task<Stream> ReadStreamAsync(string name, CancellationToken cancellationToken)
=> ReadStreamAsync(name, checksum: null, cancellationToken);
public Task<Stream> ReadStreamAsync(Project project, string name, CancellationToken cancellationToken)
=> ReadStreamAsync(project, name, checksum: null, cancellationToken);
public Task<Stream> ReadStreamAsync(Document document, string name, CancellationToken cancellationToken)
=> ReadStreamAsync(document, name, checksum: null, cancellationToken);
public Task<bool> WriteStreamAsync(string name, Stream stream, CancellationToken cancellationToken)
=> WriteStreamAsync(name, stream, checksum: null, cancellationToken);
public Task<bool> WriteStreamAsync(Project project, string name, Stream stream, CancellationToken cancellationToken)
=> WriteStreamAsync(project, name, stream, checksum: null, cancellationToken);
public Task<bool> WriteStreamAsync(Document document, string name, Stream stream, CancellationToken cancellationToken)
=> WriteStreamAsync(document, name, stream, checksum: null, cancellationToken);
}
}
src/Workspaces/Core/Portable/Workspace/Host/PersistentStorage/IChecksummedPersistentStorage.cs 0 → 100644
浏览文件 @ 3f487e01
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.CodeAnalysis.Host
{
internal interface IChecksummedPersistentStorage : IPersistentStorage
{
/// <summary>
/// Reads the existing checksum we have for the solution with the given <paramref name="name"/>,
/// or <see langword="null"/> if we do not have a checksum persisted.
/// </summary>
Task<Checksum> ReadChecksumAsync(string name, CancellationToken cancellationToken = default);
/// <summary>
/// Reads the existing checksum we have for the given <paramref name="project"/> with the given <paramref name="name"/>,
/// or <see langword="null"/> if we do not have a checksum persisted.
/// </summary>
Task<Checksum> ReadChecksumAsync(Project project, string name, CancellationToken cancellationToken = default);
/// <summary>
/// Reads the existing checksum we have for the given <paramref name="document"/> with the given <paramref name="name"/>,
/// or <see langword="null"/> if we do not have a checksum persisted.
/// </summary>
Task<Checksum> ReadChecksumAsync(Document document, string name, CancellationToken cancellationToken = default);
/// <summary>
/// Reads the stream for the solution with the given <paramref name="name"/>. If <paramref name="checksum"/>
/// is provided, the persisted checksum must match it. If there is no such stream with that name, or the
/// checksums do not match, then <see langword="null"/> will be returned.
/// </summary>
Task<Stream> ReadStreamAsync(string name, Checksum checksum = default, CancellationToken cancellationToken = default);
/// <summary>
/// Reads the stream for the <paramref name="project"/> with the given <paramref name="name"/>. If <paramref name="checksum"/>
/// is provided, the persisted checksum must match it. If there is no such stream with that name, or the
/// checksums do not match, then <see langword="null"/> will be returned.
/// </summary>
Task<Stream> ReadStreamAsync(Project project, string name, Checksum checksum = default, CancellationToken cancellationToken = default);
/// <summary>
/// Reads the stream for the <paramref name="document"/> with the given <paramref name="name"/>. If <paramref name="checksum"/>
/// is provided, the persisted checksum must match it. If there is no such stream with that name, or the
/// checksums do not match, then <see langword="null"/> will be returned.
/// </summary>
Task<Stream> ReadStreamAsync(Document document, string name, Checksum checksum = default, CancellationToken cancellationToken = default);
/// <summary>
/// Reads the stream for the solution with the given <paramref name="name"/>. An optional <paramref name="checksum"/>
/// can be provided to store along with the data. This can be used along with ReadStreamAsync with future
/// reads to ensure the data is only read back if it matches that checksum.
/// </summary>
Task<bool> WriteStreamAsync(string name, Stream stream, Checksum checksum = default, CancellationToken cancellationToken = default);
/// <summary>
/// Reads the stream for the <paramref name="project"/> with the given <paramref name="name"/>. An optional <paramref name="checksum"/>
/// can be provided to store along with the data. This can be used along with ReadStreamAsync with future
/// reads to ensure the data is only read back if it matches that checksum.
/// </summary>
Task<bool> WriteStreamAsync(Project project, string name, Stream stream, Checksum checksum = default, CancellationToken cancellationToken = default);
/// <summary>
/// Reads the stream for the <paramref name="document"/> with the given <paramref name="name"/>. An optional <paramref name="checksum"/>
/// can be provided to store along with the data. This can be used along with ReadStreamAsync with future
/// reads to ensure the data is only read back if it matches that checksum.
/// </summary>
Task<bool> WriteStreamAsync(Document document, string name, Stream stream, Checksum checksum = default, CancellationToken cancellationToken = default);
}
}
src/Workspaces/Core/Portable/Workspace/Host/PersistentStorage/IChecksummedPersistentStorageService.cs 0 → 100644
浏览文件 @ 3f487e01
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.CodeAnalysis.Host
{
internal interface IChecksummedPersistentStorageService : IPersistentStorageService2
{
new IChecksummedPersistentStorage GetStorage(Solution solution);
new IChecksummedPersistentStorage GetStorage(Solution solution, bool checkBranchId);
}
}
src/Workspaces/Core/Portable/Workspace/Host/PersistentStorage/NoOpPersistentStorage.cs
浏览文件 @ 3f487e01
......@@ -7,9 +7,9 @@
namespace Microsoft.CodeAnalysis.Host
{
internal class NoOpPersistentStorage : IPersistentStorage
internal class NoOpPersistentStorage : IChecksummedPersistentStorage
{
public static readonly IPersistentStorage Instance = new NoOpPersistentStorage();
public static readonly IChecksummedPersistentStorage Instance = new NoOpPersistentStorage();
private NoOpPersistentStorage()
{
......@@ -19,22 +19,49 @@ public void Dispose()
{
}
public Task<Stream> ReadStreamAsync(Document document, string name, CancellationToken cancellationToken = default)
public Task<Checksum> ReadChecksumAsync(string name, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Checksum>();
public Task<Checksum> ReadChecksumAsync(Project project, string name, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Checksum>();
public Task<Checksum> ReadChecksumAsync(Document document, string name, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Checksum>();
public Task<Stream> ReadStreamAsync(Document document, string name, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Stream>();
public Task<Stream> ReadStreamAsync(Project project, string name, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Stream>();
public Task<Stream> ReadStreamAsync(string name, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Stream>();
public Task<Stream> ReadStreamAsync(Project project, string name, CancellationToken cancellationToken = default)
public Task<Stream> ReadStreamAsync(string name, Checksum checksum, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Stream>();
public Task<Stream> ReadStreamAsync(string name, CancellationToken cancellationToken = default)
public Task<Stream> ReadStreamAsync(Project project, string name, Checksum checksum, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Stream>();
public Task<bool> WriteStreamAsync(Document document, string name, Stream stream, CancellationToken cancellationToken = default)
public Task<Stream> ReadStreamAsync(Document document, string name, Checksum checksum, CancellationToken cancellationToken)
=> SpecializedTasks.Default<Stream>();
public Task<bool> WriteStreamAsync(Document document, string name, Stream stream, CancellationToken cancellationToken)
=> SpecializedTasks.False;
public Task<bool> WriteStreamAsync(Project project, string name, Stream stream, CancellationToken cancellationToken)
=> SpecializedTasks.False;
public Task<bool> WriteStreamAsync(string name, Stream stream, CancellationToken cancellationToken)
=> SpecializedTasks.False;
public Task<bool> WriteStreamAsync(string name, Stream stream, Checksum checksum, CancellationToken cancellationToken)
=> SpecializedTasks.False;
public Task<bool> WriteStreamAsync(Project project, string name, Stream stream, CancellationToken cancellationToken = default)
public Task<bool> WriteStreamAsync(Project project, string name, Stream stream, Checksum checksum, CancellationToken cancellationToken)
=> SpecializedTasks.False;
public Task<bool> WriteStreamAsync(string name, Stream stream, CancellationToken cancellationToken = default)
public Task<bool> WriteStreamAsync(Document document, string name, Stream stream, Checksum checksum, CancellationToken cancellationToken)
=> SpecializedTasks.False;
}
}
src/Workspaces/Core/Portable/Workspace/Host/PersistentStorage/NoOpPersistentStorageService.cs
浏览文件 @ 3f487e01
......@@ -2,7 +2,7 @@
namespace Microsoft.CodeAnalysis.Host
{
internal class NoOpPersistentStorageService : IPersistentStorageService2
internal class NoOpPersistentStorageService : IChecksummedPersistentStorageService
{
public static readonly IPersistentStorageService Instance = new NoOpPersistentStorageService();
......@@ -15,5 +15,11 @@ public IPersistentStorage GetStorage(Solution solution)
public IPersistentStorage GetStorage(Solution solution, bool checkBranchId)
=> NoOpPersistentStorage.Instance;
IChecksummedPersistentStorage IChecksummedPersistentStorageService.GetStorage(Solution solution)
=> NoOpPersistentStorage.Instance;
IChecksummedPersistentStorage IChecksummedPersistentStorageService.GetStorage(Solution solution, bool checkBranchId)
=> NoOpPersistentStorage.Instance;
}
}
src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs
浏览文件 @ 3f487e01
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Runtime.InteropServices;
using System.Threading;
using Microsoft.CodeAnalysis.PooledObjects;
using Microsoft.CodeAnalysis.Serialization;
using Roslyn.Utilities;
using System.Diagnostics;
namespace Microsoft.CodeAnalysis
{
// various factory methods.
// all these are just helper methods
// various factory methods. all these are just helper methods
internal partial class Checksum
{
private static readonly ObjectPool<IncrementalHash> s_incrementalHashPool =
new ObjectPool<IncrementalHash>(() => IncrementalHash.CreateHash(HashAlgorithmName.SHA256), size: 20);
public static Checksum Create(string val)
{
using (var pooledHash = s_incrementalHashPool.GetPooledObject())
using (var pooledBuffer = SharedPools.ByteArray.GetPooledObject())
{
var hash = pooledHash.Object;
var stringBytes = MemoryMarshal.AsBytes(val.AsSpan());
Debug.Assert(stringBytes.Length == val.Length * 2);
var buffer = pooledBuffer.Object;
var index = 0;
while (index < stringBytes.Length)
{
var remaining = stringBytes.Length - index;
var toCopy = Math.Min(remaining, buffer.Length);
stringBytes.Slice(index, toCopy).CopyTo(buffer);
hash.AppendData(buffer, 0, toCopy);
index += toCopy;
}
return From(hash.GetHashAndReset());
}
}
public static Checksum Create(Stream stream)
{
using (var pooledHash = s_incrementalHashPool.GetPooledObject())
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或