提交 b6e09b1a 编写于 作者: T Tomáš Matoušek

Merge pull request #3920 from tmat/BlobHeapSerialization

Defer building blob heap to metadata serialization phase
......@@ -40,7 +40,7 @@ internal sealed class DeltaMetadataWriter : MetadataWriter
private readonly InstanceAndStructuralReferenceIndex<IGenericMethodInstanceReference> _methodSpecIndex;
private readonly HeapOrReferenceIndex<ITypeReference> _typeRefIndex;
private readonly InstanceAndStructuralReferenceIndex<ITypeReference> _typeSpecIndex;
private readonly HeapOrReferenceIndex<int> _standAloneSignatureIndex;
private readonly HeapOrReferenceIndex<BlobIdx> _standAloneSignatureIndex;
private readonly Dictionary<IMethodDefinition, AddedOrChangedMethodInfo> _addedOrChangedMethods;
public DeltaMetadataWriter(
......@@ -82,7 +82,7 @@ internal sealed class DeltaMetadataWriter : MetadataWriter
_methodSpecIndex = new InstanceAndStructuralReferenceIndex<IGenericMethodInstanceReference>(this, new MethodSpecComparer(this), lastRowId: sizes[(int)TableIndex.MethodSpec]);
_typeRefIndex = new HeapOrReferenceIndex<ITypeReference>(this, lastRowId: sizes[(int)TableIndex.TypeRef]);
_typeSpecIndex = new InstanceAndStructuralReferenceIndex<ITypeReference>(this, new TypeSpecComparer(this), lastRowId: sizes[(int)TableIndex.TypeSpec]);
_standAloneSignatureIndex = new HeapOrReferenceIndex<int>(this, lastRowId: sizes[(int)TableIndex.StandAloneSig]);
_standAloneSignatureIndex = new HeapOrReferenceIndex<BlobIdx>(this, lastRowId: sizes[(int)TableIndex.StandAloneSig]);
_addedOrChangedMethods = new Dictionary<IMethodDefinition, AddedOrChangedMethodInfo>();
}
......@@ -402,12 +402,12 @@ protected override IReadOnlyList<ITypeReference> GetTypeSpecs()
return _typeSpecIndex.Rows;
}
protected override int GetOrAddStandAloneSignatureIndex(int blobIndex)
protected override int GetOrAddStandAloneSignatureIndex(BlobIdx blobIndex)
{
return _standAloneSignatureIndex.GetOrAdd(blobIndex);
}
protected override IReadOnlyList<int> GetStandAloneSignatures()
protected override IReadOnlyList<BlobIdx> GetStandAloneSignatures()
{
return _standAloneSignatureIndex.Rows;
}
......@@ -630,7 +630,7 @@ protected override int SerializeLocalVariablesSignature(IMethodBody body)
encInfos.Add(CreateEncLocalInfo(local, signature));
}
int blobIndex = heaps.GetBlobIndex(writer);
BlobIdx blobIndex = heaps.GetBlobIndex(writer);
localSignatureRowId = GetOrAddStandAloneSignatureIndex(blobIndex);
writer.Free();
}
......
......@@ -591,6 +591,10 @@ internal void WriteCompressedSignedInteger(int value)
}
}
private const int SingleByteCompressedIntegerMaxValue = 0x7f;
private const int TwoByteCompressedIntegerMaxValue = 0x3fff;
private const int MaxCompressedIntegerValue = 0x1fffffff;
/// <summary>
/// Implements compressed unsigned integer encoding as defined by ECMA-335-II chapter 23.2: Blobs and signatures.
/// </summary>
......@@ -605,21 +609,21 @@ internal void WriteCompressedSignedInteger(int value)
/// </remarks>
internal void WriteCompressedUInt(uint val)
{
Debug.Assert(val <= MaxCompressedIntegerValue);
unchecked
{
if (val <= 0x7f)
if (val <= SingleByteCompressedIntegerMaxValue)
{
WriteByte((byte)val);
}
else if (val <= 0x3fff)
else if (val <= TwoByteCompressedIntegerMaxValue)
{
WriteByte((byte)(0x80 | (val >> 8)));
WriteByte((byte)val);
}
else
{
Debug.Assert(val <= 0x1fffffff);
WriteByte((byte)(0xc0 | (val >> 24)));
WriteByte((byte)(val >> 16));
WriteByte((byte)(val >> 8));
......@@ -628,6 +632,23 @@ internal void WriteCompressedUInt(uint val)
}
}
internal static int GetCompressedIntegerSize(int value)
{
Debug.Assert(value <= MaxCompressedIntegerValue);
if (value <= SingleByteCompressedIntegerMaxValue)
{
return 1;
}
if (value <= TwoByteCompressedIntegerMaxValue)
{
return 2;
}
return 4;
}
/// <summary>
/// Writes a constant value (see ECMA-335 Partition II section 22.9) at the current position.
/// </summary>
......
......@@ -30,7 +30,7 @@ internal sealed class FullMetadataWriter : MetadataWriter
private readonly InstanceAndStructuralReferenceIndex<IGenericMethodInstanceReference> _methodSpecIndex;
private readonly HeapOrReferenceIndex<ITypeReference> _typeRefIndex;
private readonly InstanceAndStructuralReferenceIndex<ITypeReference> _typeSpecIndex;
private readonly HeapOrReferenceIndex<int> _standAloneSignatureIndex;
private readonly HeapOrReferenceIndex<BlobIdx> _standAloneSignatureIndex;
public static MetadataWriter Create(
EmitContext context,
......@@ -82,7 +82,7 @@ internal sealed class FullMetadataWriter : MetadataWriter
_methodSpecIndex = new InstanceAndStructuralReferenceIndex<IGenericMethodInstanceReference>(this, new MethodSpecComparer(this));
_typeRefIndex = new HeapOrReferenceIndex<ITypeReference>(this);
_typeSpecIndex = new InstanceAndStructuralReferenceIndex<ITypeReference>(this, new TypeSpecComparer(this));
_standAloneSignatureIndex = new HeapOrReferenceIndex<int>(this);
_standAloneSignatureIndex = new HeapOrReferenceIndex<BlobIdx>(this);
}
protected override ushort Generation
......@@ -265,12 +265,12 @@ protected override IReadOnlyList<ITypeReference> GetTypeSpecs()
return _typeSpecIndex.Rows;
}
protected override int GetOrAddStandAloneSignatureIndex(int blobIndex)
protected override int GetOrAddStandAloneSignatureIndex(BlobIdx blobIndex)
{
return _standAloneSignatureIndex.GetOrAdd(blobIndex);
}
protected override IReadOnlyList<int> GetStandAloneSignatures()
protected override IReadOnlyList<BlobIdx> GetStandAloneSignatures()
{
return _standAloneSignatureIndex.Rows;
}
......
......@@ -34,15 +34,15 @@ public bool Equals(ITypeMemberReference x, ITypeMemberReference y)
return false;
}
IFieldReference/*?*/ xf = x as IFieldReference;
IFieldReference/*?*/ yf = y as IFieldReference;
var xf = x as IFieldReference;
var yf = y as IFieldReference;
if (xf != null && yf != null)
{
return _metadataWriter.GetFieldSignatureIndex(xf) == _metadataWriter.GetFieldSignatureIndex(yf);
}
IMethodReference/*?*/ xm = x as IMethodReference;
IMethodReference/*?*/ ym = y as IMethodReference;
var xm = x as IMethodReference;
var ym = y as IMethodReference;
if (xm != null && ym != null)
{
return _metadataWriter.GetMethodSignatureIndex(xm) == _metadataWriter.GetMethodSignatureIndex(ym);
......@@ -55,17 +55,17 @@ public int GetHashCode(ITypeMemberReference memberRef)
{
int hash = Hash.Combine(memberRef.Name, (int)_metadataWriter.GetMemberRefParentCodedIndex(memberRef) << 4);
IFieldReference/*?*/ fieldRef = memberRef as IFieldReference;
var fieldRef = memberRef as IFieldReference;
if (fieldRef != null)
{
hash = Hash.Combine(hash, (int)_metadataWriter.GetFieldSignatureIndex(fieldRef));
hash = Hash.Combine(hash, _metadataWriter.GetFieldSignatureIndex(fieldRef).GetHashCode());
}
else
{
IMethodReference/*?*/ methodRef = memberRef as IMethodReference;
var methodRef = memberRef as IMethodReference;
if (methodRef != null)
{
hash = Hash.Combine(hash, (int)_metadataWriter.GetMethodSignatureIndex(methodRef));
hash = Hash.Combine(hash, _metadataWriter.GetMethodSignatureIndex(methodRef).GetHashCode());
}
}
......
......@@ -14,16 +14,81 @@
namespace Microsoft.Cci
{
/// <summary>
/// Wraps a virtual string table index.
/// An override to SerializeIndex does the resolving at the right time.
/// Represents a value on #String heap that has not been serialized yet.
/// </summary>
internal struct StringIdx
internal struct StringIdx : IEquatable<StringIdx>
{
public readonly int VirtIdx;
// index in _stringIndexToHeapPositionMap
public readonly int MapIndex;
internal StringIdx(int virtIdx)
{
this.VirtIdx = virtIdx;
MapIndex = virtIdx;
}
public bool Equals(StringIdx other)
{
return MapIndex == other.MapIndex;
}
public override bool Equals(object obj)
{
return obj is StringIdx && Equals((StringIdx)obj);
}
public override int GetHashCode()
{
return MapIndex.GetHashCode();
}
public static bool operator ==(StringIdx left, StringIdx right)
{
return left.Equals(right);
}
public static bool operator !=(StringIdx left, StringIdx right)
{
return !left.Equals(right);
}
}
/// <summary>
/// Represents a value on #Blob heap that has not been serialized yet.
/// </summary>
internal struct BlobIdx : IEquatable<BlobIdx>
{
// The position of the blob on heap relative to the start of the heap.
// In EnC deltas this value is not the same as the value stored in blob token.
public readonly int HeapPosition;
internal BlobIdx(int heapPosition)
{
HeapPosition = heapPosition;
}
public bool Equals(BlobIdx other)
{
return HeapPosition == other.HeapPosition;
}
public override bool Equals(object obj)
{
return obj is BlobIdx && Equals((BlobIdx)obj);
}
public override int GetHashCode()
{
return HeapPosition.GetHashCode();
}
public static bool operator ==(BlobIdx left, BlobIdx right)
{
return left.Equals(right);
}
public static bool operator !=(BlobIdx left, BlobIdx right)
{
return !left.Equals(right);
}
}
......@@ -32,75 +97,76 @@ internal sealed class MetadataHeapsBuilder
private static readonly Encoding s_utf8Encoding = Encoding.UTF8;
// #US heap
private readonly Dictionary<string, int> _userStringIndex = new Dictionary<string, int>();
private readonly Dictionary<string, int> _userStrings = new Dictionary<string, int>();
private readonly BlobWriter _userStringWriter = new BlobWriter(1024);
private readonly int _userStringIndexStartOffset;
private readonly int _userStringHeapStartOffset;
// #String heap
private Dictionary<string, StringIdx> _stringIndex = new Dictionary<string, StringIdx>(128);
private int[] _stringIndexMap;
private readonly BlobWriter _stringWriter = new BlobWriter(1024);
private readonly int _stringIndexStartOffset;
private Dictionary<string, StringIdx> _strings = new Dictionary<string, StringIdx>(128);
private int[] _stringIndexToHeapPositionMap;
private BlobWriter _stringWriter;
private readonly int _stringHeapStartOffset;
// #Blob heap
private readonly Dictionary<ImmutableArray<byte>, int> _blobIndex = new Dictionary<ImmutableArray<byte>, int>(ByteSequenceComparer.Instance);
private readonly BlobWriter _blobWriter = new BlobWriter(1024);
private readonly int _blobIndexStartOffset;
private readonly Dictionary<ImmutableArray<byte>, BlobIdx> _blobs = new Dictionary<ImmutableArray<byte>, BlobIdx>(ByteSequenceComparer.Instance);
private readonly int _blobHeapStartOffset;
private int _blobHeapSize;
// #GUID heap
private readonly Dictionary<Guid, int> _guidIndex = new Dictionary<Guid, int>();
private readonly Dictionary<Guid, int> _guids = new Dictionary<Guid, int>();
private readonly BlobWriter _guidWriter = new BlobWriter(16); // full metadata has just a single guid
private bool _streamsAreComplete;
public MetadataHeapsBuilder(
int userStringIndexStartOffset = 0,
int stringIndexStartOffset = 0,
int blobIndexStartOffset = 0,
int guidIndexStartOffset = 0)
int userStringHeapStartOffset = 0,
int stringHeapStartOffset = 0,
int blobHeapStartOffset = 0,
int guidHeapStartOffset = 0)
{
// Add zero-th entry to heaps.
// Full metadata represent empty blob/string at heap index 0.
// Delta metadata requires these to avoid nil generation-relative handles,
// which are technically viable but confusing.
_blobWriter.WriteByte(0);
_stringWriter.WriteByte(0);
_userStringWriter.WriteByte(0);
_blobs.Add(ImmutableArray<byte>.Empty, new BlobIdx(0));
_blobHeapSize = 1;
// When EnC delta is applied #US, #String and #Blob heaps are appended.
// Thus indices of strings and blobs added to this generation are offset
// by the sum of respective heap sizes of all previous generations.
_userStringIndexStartOffset = userStringIndexStartOffset;
_stringIndexStartOffset = stringIndexStartOffset;
_blobIndexStartOffset = blobIndexStartOffset;
_userStringHeapStartOffset = userStringHeapStartOffset;
_stringHeapStartOffset = stringHeapStartOffset;
_blobHeapStartOffset = blobHeapStartOffset;
// Unlike other heaps, #Guid heap in EnC delta is zero-padded.
_guidWriter.Pad(guidIndexStartOffset);
_guidWriter.Pad(guidHeapStartOffset);
}
internal int GetBlobIndex(BlobWriter stream)
internal BlobIdx GetBlobIndex(BlobWriter stream)
{
// TODO: avoid making a copy if the blob exists in the index
return GetBlobIndex(stream.ToImmutableArray());
}
internal int GetBlobIndex(ImmutableArray<byte> blob)
internal BlobIdx GetBlobIndex(ImmutableArray<byte> blob)
{
int result = 0;
if (blob.Length == 0 || _blobIndex.TryGetValue(blob, out result))
BlobIdx index;
if (!_blobs.TryGetValue(blob, out index))
{
return result;
Debug.Assert(!_streamsAreComplete);
index = new BlobIdx(_blobHeapSize);
_blobs.Add(blob, index);
_blobHeapSize += BlobWriter.GetCompressedIntegerSize(blob.Length) + blob.Length;
}
Debug.Assert(!_streamsAreComplete);
result = _blobWriter.Position + _blobIndexStartOffset;
_blobIndex.Add(blob, result);
_blobWriter.WriteCompressedUInt((uint)blob.Length);
_blobWriter.WriteBytes(blob);
return result;
return index;
}
public int GetConstantBlobIndex(object value)
public BlobIdx GetConstantBlobIndex(object value)
{
string str = value as string;
if (str != null)
......@@ -113,7 +179,7 @@ public int GetConstantBlobIndex(object value)
return this.GetBlobIndex(writer);
}
public int GetBlobIndex(string str)
public BlobIdx GetBlobIndex(string str)
{
byte[] byteArray = new byte[str.Length * 2];
int i = 0;
......@@ -134,7 +200,7 @@ public int GetGuidIndex(Guid guid)
}
int result;
if (_guidIndex.TryGetValue(guid, out result))
if (_guids.TryGetValue(guid, out result))
{
return result;
}
......@@ -158,24 +224,12 @@ public int AllocateGuid(Guid guid)
// Its first element is numbered 1, its second 2, and so on.
int result = (_guidWriter.Length >> 4) + 1;
_guidIndex.Add(guid, result);
_guids.Add(guid, result);
_guidWriter.WriteBytes(guid.ToByteArray());
return result;
}
public unsafe byte[] GetExistingBlob(int signatureOffset)
{
fixed (byte* ptr = _blobWriter.Buffer)
{
var reader = new BlobReader(ptr + signatureOffset, (int)_blobWriter.Length + _blobIndexStartOffset - signatureOffset);
int size;
bool isValid = reader.TryReadCompressedInteger(out size);
Debug.Assert(isValid);
return reader.ReadBytes(size);
}
}
public StringIdx GetStringIndex(string str)
{
StringIdx index;
......@@ -183,11 +237,11 @@ public StringIdx GetStringIndex(string str)
{
index = new StringIdx(0);
}
else if (!_stringIndex.TryGetValue(str, out index))
else if (!_strings.TryGetValue(str, out index))
{
Debug.Assert(!_streamsAreComplete);
index = new StringIdx(_stringIndex.Count + 1); // idx 0 is reserved for empty string
_stringIndex.Add(str, index);
index = new StringIdx(_strings.Count + 1); // idx 0 is reserved for empty string
_strings.Add(str, index);
}
return index;
......@@ -195,17 +249,22 @@ public StringIdx GetStringIndex(string str)
public int ResolveStringIndex(StringIdx index)
{
return _stringIndexMap[index.VirtIdx];
return _stringHeapStartOffset + _stringIndexToHeapPositionMap[index.MapIndex];
}
public int ResolveBlobIndex(BlobIdx index)
{
return _blobHeapStartOffset + index.HeapPosition;
}
public int GetUserStringToken(string str)
{
int index;
if (!_userStringIndex.TryGetValue(str, out index))
if (!_userStrings.TryGetValue(str, out index))
{
Debug.Assert(!_streamsAreComplete);
index = _userStringWriter.Position + _userStringIndexStartOffset;
_userStringIndex.Add(str, index);
index = _userStringWriter.Position + _userStringHeapStartOffset;
_userStrings.Add(str, index);
_userStringWriter.WriteCompressedUInt((uint)str.Length * 2 + 1);
_userStringWriter.WriteUTF16(str);
......@@ -276,10 +335,10 @@ public ImmutableArray<int> GetHeapSizes()
{
var heapSizes = new int[MetadataTokens.HeapCount];
heapSizes[(int)HeapIndex.UserString] = (int)_userStringWriter.Length;
heapSizes[(int)HeapIndex.String] = (int)_stringWriter.Length;
heapSizes[(int)HeapIndex.Blob] = (int)_blobWriter.Length;
heapSizes[(int)HeapIndex.Guid] = (int)_guidWriter.Length;
heapSizes[(int)HeapIndex.UserString] = _userStringWriter.Length;
heapSizes[(int)HeapIndex.String] = _stringWriter.Length;
heapSizes[(int)HeapIndex.Blob] = _blobHeapSize;
heapSizes[(int)HeapIndex.Guid] = _guidWriter.Length;
return ImmutableArray.CreateRange(heapSizes);
}
......@@ -291,34 +350,38 @@ public ImmutableArray<int> GetHeapSizes()
private void SerializeStringHeap()
{
// Sort by suffix and remove stringIndex
var sorted = new List<KeyValuePair<string, StringIdx>>(_stringIndex);
var sorted = new List<KeyValuePair<string, StringIdx>>(_strings);
sorted.Sort(new SuffixSort());
_stringIndex = null;
_strings = null;
_stringWriter = new BlobWriter(1024);
// Create VirtIdx to Idx map and add entry for empty string
_stringIndexMap = new int[sorted.Count + 1];
_stringIndexMap[0] = 0;
_stringIndexToHeapPositionMap = new int[sorted.Count + 1];
_stringIndexToHeapPositionMap[0] = 0;
_stringWriter.WriteByte(0);
// Find strings that can be folded
string prev = string.Empty;
foreach (KeyValuePair<string, StringIdx> cur in sorted)
foreach (KeyValuePair<string, StringIdx> entry in sorted)
{
int position = _stringWriter.Position + _stringIndexStartOffset;
int position = _stringWriter.Position;
// It is important to use ordinal comparison otherwise we'll use the current culture!
if (prev.EndsWith(cur.Key, StringComparison.Ordinal))
if (prev.EndsWith(entry.Key, StringComparison.Ordinal))
{
// Map over the tail of prev string. Watch for null-terminator of prev string.
_stringIndexMap[cur.Value.VirtIdx] = position - (s_utf8Encoding.GetByteCount(cur.Key) + 1);
_stringIndexToHeapPositionMap[entry.Value.MapIndex] = position - (s_utf8Encoding.GetByteCount(entry.Key) + 1);
}
else
{
_stringIndexMap[cur.Value.VirtIdx] = position;
_stringWriter.WriteString(cur.Key, s_utf8Encoding);
_stringIndexToHeapPositionMap[entry.Value.MapIndex] = position;
_stringWriter.WriteString(entry.Key, s_utf8Encoding);
_stringWriter.WriteByte(0);
}
prev = cur.Key;
prev = entry.Key;
}
}
......@@ -350,15 +413,43 @@ public int Compare(KeyValuePair<string, StringIdx> xPair, KeyValuePair<string, S
}
}
public void WriteTo(BlobWriter stream, out int guidHeapStartOffset)
public void WriteTo(BlobWriter writer, out int guidHeapStartOffset)
{
WriteAligned(_stringWriter, stream);
WriteAligned(_userStringWriter, stream);
WriteAligned(_stringWriter, writer);
WriteAligned(_userStringWriter, writer);
guidHeapStartOffset = writer.Position;
WriteAligned(_guidWriter, writer);
WriteAlignedBlobHeap(writer);
}
private void WriteAlignedBlobHeap(BlobWriter writer)
{
int heapStart = writer.Position;
// ensure enough space in the buffer:
writer.Position += _blobHeapSize;
// Perf consideration: With large heap the following loop may cause a lot of cache misses
// since the order of entries in _blobs dictionary depends on the hash of the array values,
// which is not correlated to the heap index. If we observe such issue we should order
// the entries by heap position before running this loop.
foreach (var entry in _blobs)
{
int heapOffset = entry.Value.HeapPosition;
var blob = entry.Key;
writer.Position = heapStart + heapOffset;
writer.WriteCompressedUInt((uint)blob.Length);
writer.WriteBytes(blob);
}
guidHeapStartOffset = stream.Position;
Debug.Assert(writer.Length - heapStart == _blobHeapSize);
WriteAligned(_guidWriter, stream);
WriteAligned(_blobWriter, stream);
// add padding:
writer.Position = writer.Length;
writer.Write(0, BitArithmeticUtilities.Align(_blobHeapSize, 4) - _blobHeapSize);
}
private static void WriteAligned(BlobWriter source, BlobWriter target)
......
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Collections.Generic;
using Roslyn.Utilities;
namespace Microsoft.Cci
{
......@@ -27,8 +28,9 @@ public bool Equals(IGenericMethodInstanceReference x, IGenericMethodInstanceRefe
public int GetHashCode(IGenericMethodInstanceReference methodInstanceReference)
{
return (int)((_metadataWriter.GetMethodDefOrRefCodedIndex(methodInstanceReference.GetGenericMethod(_metadataWriter.Context)) << 2) ^
_metadataWriter.GetMethodInstanceSignatureIndex(methodInstanceReference));
return Hash.Combine(
(int)_metadataWriter.GetMethodDefOrRefCodedIndex(methodInstanceReference.GetGenericMethod(_metadataWriter.Context)),
_metadataWriter.GetMethodInstanceSignatureIndex(methodInstanceReference).GetHashCode());
}
}
}
......@@ -15,12 +15,12 @@ internal TypeSpecComparer(MetadataWriter metadataWriter)
public bool Equals(ITypeReference x, ITypeReference y)
{
return x == y || _metadataWriter.GetTypeSpecSignatureIndex(x) == _metadataWriter.GetTypeSpecSignatureIndex(y);
return x == y || _metadataWriter.GetTypeSpecSignatureIndex(x).Equals(_metadataWriter.GetTypeSpecSignatureIndex(y));
}
public int GetHashCode(ITypeReference typeReference)
{
return (int)_metadataWriter.GetTypeSpecSignatureIndex(typeReference);
return _metadataWriter.GetTypeSpecSignatureIndex(typeReference).GetHashCode();
}
}
}
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册