Make the PDBs deterministic by hashing a log of the

sequence of operations to the PDB writing library
Fixes #926
Closes #2342

src/Compilers/CSharp/Test/Emit/Emit/DeterministicTests.cs

@@ -15,12 +15,12 @@ namespace Microsoft.CodeAnalysis.CSharp.UnitTests.Emit
 {
     public class DeterministicTests : EmitMetadataTestBase
     {
-        private Guid CompiledGuid(string source, string assemblyName)
+        private Guid CompiledGuid(string source, string assemblyName, bool debug)
         {
             var compilation = CreateCompilation(source,
                 assemblyName: assemblyName,
                 references: new[] { MscorlibRef },
-                options: TestOptions.ReleaseExe.WithFeatures(ImmutableArray.Create("deterministic")));
+                options: (debug ? TestOptions.DebugExe : TestOptions.ReleaseExe).WithFeatures(ImmutableArray.Create("deterministic")));
 
             Guid result = default(Guid);
             base.CompileAndVerify(compilation, emitters: TestEmitters.CCI, validator: (a, eo) =>
@@ -54,13 +54,27 @@ public void Simple()
 {
     public static void Main(string[] args) {}
 }";
-            var mvid1 = CompiledGuid(source, "X1");
-            var mvid2 = CompiledGuid(source, "X1");
-            var mvid3 = CompiledGuid(source, "X2");
-            var mvid4 = CompiledGuid(source, "X2");
+            // Two identical compilations should produce the same MVID
+            var mvid1 = CompiledGuid(source, "X1", false);
+            var mvid2 = CompiledGuid(source, "X1", false);
             Assert.Equal(mvid1, mvid2);
-            Assert.Equal(mvid3, mvid4);
+
+            // Changing the module name should change the MVID
+            var mvid3 = CompiledGuid(source, "X2", false);
             Assert.NotEqual(mvid1, mvid3);
+
+            // Two identical debug compilations should produce the same MVID also
+            var mvid5 = CompiledGuid(source, "X1", true);
+            var mvid6 = CompiledGuid(source, "X1", true);
+            Assert.Equal(mvid5, mvid6);
+
+            // But even in debug, a changed module name changes the MVID
+            var mvid7 = CompiledGuid(source, "X2", true);
+            Assert.NotEqual(mvid5, mvid7);
+
+            // adding the debug option should change the MVID
+            Assert.NotEqual(mvid1, mvid5);
+            Assert.NotEqual(mvid3, mvid7);
         }
 
         [Fact]

src/Compilers/Core/Portable/InternalUtilities/HashAlgorithms.cs

@@ -12,9 +12,12 @@ namespace Roslyn.Utilities
 
     internal abstract class HashAlgorithm : IDisposable
     {
-        private static readonly MethodInfo s_bytesMethod;
-        private static readonly MethodInfo s_bytesOffsetCountMethod;
-        private static readonly MethodInfo s_streamMethod;
+        private static readonly MethodInfo s_ComputeHash_bytes_Method;
+        private static readonly MethodInfo s_ComputeHash_bytesOffsetCount_Method;
+        private static readonly MethodInfo s_ComputeHash_stream_Method;
+        private static readonly MethodInfo s_TransformBlock_Method;
+        private static readonly MethodInfo s_TransformFinalBlock_Method;
+        private static readonly MethodInfo s_Hash_PropertyGetter;
 
         private readonly IDisposable _hashInstance;
 
@@ -30,20 +33,44 @@ static HashAlgorithm()
             {
                 var methods = type.GetTypeInfo().GetDeclaredMethods("ComputeHash");
 
-                s_bytesMethod = (from m in methods
+                // https://msdn.microsoft.com/en-us/library/s02tk69a(v=vs.110).aspx
+                s_ComputeHash_bytes_Method = (from m in methods
                                  let ps = m.GetParameters()
                                  where ps.Length == 1 && ps[0].ParameterType == typeof(byte[])
                                  select m).Single();
 
-                s_bytesOffsetCountMethod = (from m in methods
+                // https://msdn.microsoft.com/en-us/library/1e59xaaz(v=vs.110).aspx
+                s_ComputeHash_bytesOffsetCount_Method = (from m in methods
                                             let ps = m.GetParameters()
                                             where ps.Length == 3 && ps[0].ParameterType == typeof(byte[]) && ps[1].ParameterType == typeof(int) && ps[2].ParameterType == typeof(int)
                                             select m).Single();
 
-                s_streamMethod = (from m in methods
+                // https://msdn.microsoft.com/en-us/library/xa627k19(v=vs.110).aspx
+                s_ComputeHash_stream_Method = (from m in methods
                                   let ps = m.GetParameters()
                                   where ps.Length == 1 && ps[0].ParameterType == typeof(Stream)
                                   select m).Single();
+
+                // https://msdn.microsoft.com/en-us/library/system.security.cryptography.hashalgorithm.transformblock(v=vs.110).aspx
+                s_TransformBlock_Method = (from m in type.GetTypeInfo().GetDeclaredMethods("TransformBlock")
+                                          let ps = m.GetParameters()
+                                          where ps.Length == 5 && ps[0].ParameterType == typeof(byte[]) &&
+                                                                  ps[1].ParameterType == typeof(int) &&
+                                                                  ps[2].ParameterType == typeof(int) &&
+                                                                  ps[3].ParameterType == typeof(byte[]) &&
+                                                                  ps[4].ParameterType == typeof(int)
+                                          select m).Single();
+
+                // https://msdn.microsoft.com/en-us/library/system.security.cryptography.hashalgorithm.transformblock(v=vs.110).aspx
+                s_TransformFinalBlock_Method = (from m in type.GetTypeInfo().GetDeclaredMethods("TransformFinalBlock")
+                                          let ps = m.GetParameters()
+                                          where ps.Length == 3 && ps[0].ParameterType == typeof(byte[]) &&
+                                                                  ps[1].ParameterType == typeof(int) &&
+                                                                  ps[2].ParameterType == typeof(int)
+                                          select m).Single();
+
+                // https://msdn.microsoft.com/en-us/library/system.security.cryptography.hashalgorithm.hash(v=vs.110).aspx
+                s_Hash_PropertyGetter = type.GetTypeInfo().GetDeclaredProperty("Hash").GetMethod;
             }
         }
 
@@ -90,19 +117,41 @@ protected HashAlgorithm(IDisposable hashInstance)
 
         public byte[] ComputeHash(byte[] bytes)
         {
-            return (byte[])s_bytesMethod.Invoke(_hashInstance, new object[] { bytes });
+            return (byte[])s_ComputeHash_bytes_Method.Invoke(_hashInstance, new object[] { bytes });
         }
 
         public byte[] ComputeHash(byte[] bytes, int offset, int count)
         {
-            return (byte[])s_bytesOffsetCountMethod.Invoke(_hashInstance, new object[] { bytes, offset, count });
+            return (byte[])s_ComputeHash_bytesOffsetCount_Method.Invoke(_hashInstance, new object[] { bytes, offset, count });
         }
 
         public byte[] ComputeHash(Stream stream)
         {
-            return (byte[])s_streamMethod.Invoke(_hashInstance, new object[] { stream });
+            return (byte[])s_ComputeHash_stream_Method.Invoke(_hashInstance, new object[] { stream });
+        }
+
+        /// <summary>
+        /// Invoke the underlying HashAlgorithm's TransformBlock operation on the provided data.
+        /// </summary>
+        public void TransformBlock(byte[] inputBuffer, int inputCount)
+        {
+            int inputOffset = 0;
+            while (inputCount > 0)
+            {
+                int written = (int)s_TransformBlock_Method.Invoke(_hashInstance, new object[] { inputBuffer, inputOffset, inputCount, inputBuffer, inputOffset });
+                Debug.Assert(inputCount == written); // does the TransformBlock method always consume the complete data given to it?
+                inputCount -= written;
+                inputOffset += written;
+            }
+        }
+
+        public void TransformFinalBlock(byte[] inputBuffer, int inputCount)
+        {
+            s_TransformFinalBlock_Method.Invoke(_hashInstance, new object[] { inputBuffer, 0, inputCount });
         }
 
+        public byte[] Hash => (byte[])s_Hash_PropertyGetter.Invoke(_hashInstance, new object[] { });
+
         public void Dispose()
         {
             _hashInstance.Dispose();

src/Compilers/Core/Portable/PEWriter/ContentId.cs

@@ -18,12 +18,13 @@ public ContentId(byte[] guid, byte[] stamp)
             Stamp = stamp;
         }
 
-        internal static ContentId FromSha1(ImmutableArray<byte> sha1)
+        internal static ContentId FromHash(ImmutableArray<byte> hashCode)
         {
+            Debug.Assert(hashCode.Length >= 20);
             var guid = new byte[16];
             for (var i = 0; i < guid.Length; i++)
             {
-                guid[i] = sha1[i];
+                guid[i] = hashCode[i];
             }
 
             // modify the guid data so it decodes to the form of a "random" guid ala rfc4122
@@ -36,10 +37,10 @@ internal static ContentId FromSha1(ImmutableArray<byte> sha1)
 
             // compute a random-looking stamp from the remaining bits, but with the upper bit set
             var stamp = new byte[4];
-            stamp[0] = sha1[16];
-            stamp[1] = sha1[17];
-            stamp[2] = sha1[18];
-            stamp[3] = (byte)(sha1[19] | 0x80);
+            stamp[0] = hashCode[16];
+            stamp[1] = hashCode[17];
+            stamp[2] = hashCode[18];
+            stamp[3] = (byte)(hashCode[19] | 0x80);
 
             return new ContentId(guid, stamp);
         }

src/Compilers/Core/Portable/PEWriter/PeWriter.cs

@@ -140,7 +140,7 @@ private bool WritePeToStream(MetadataWriter mdWriter, Func<Stream> getPeStream,
             else
             {
                 pdbContentId = default(ContentId);
-            } 
+            }
 
             FillInSectionHeaders();
 
@@ -159,7 +159,7 @@ private bool WritePeToStream(MetadataWriter mdWriter, Func<Stream> getPeStream,
             long metadataPosition;
 
             WriteHeaders(peStream, out ntHeaderTimestampPosition);
-            
+
             WriteTextSection(
                 peStream,
                 corHeader,
@@ -211,10 +211,10 @@ private int CalculateMappedFieldDataStreamRva(MetadataSizes metadataSizes)
             Debug.Assert(ntHeaderTimestampPosition != 0);
 
             var previousPosition = peStream.Position;
-           
+
             // Compute and write deterministic guid data over the relevant portion of the stream
             peStream.Position = 0;
-            var contentId = ContentId.FromSha1(CryptographicHashProvider.ComputeSha1(peStream));
+            var contentId = ContentId.FromHash(CryptographicHashProvider.ComputeSha1(peStream));
 
             // The existing Guid should be zero.
             CheckZeroDataInStream(peStream, mvidPosition, contentId.Guid.Length);
@@ -267,7 +267,7 @@ private int ComputeStrongNameSignatureSize()
         private int ComputeOffsetToDebugTable(MetadataSizes metadataSizes)
         {
             return
-                ComputeOffsetToMetadata(metadataSizes.ILStreamSize) + 
+                ComputeOffsetToMetadata(metadataSizes.ILStreamSize) +
                 metadataSizes.MetadataSize +
                 metadataSizes.ResourceDataSize +
                 ComputeStrongNameSignatureSize(); // size of strong name hash
@@ -283,10 +283,10 @@ private int ComputeOffsetToImportTable(MetadataSizes metadataSizes)
 
         private int ComputeOffsetToMetadata(int ilStreamLength)
         {
-            return 
-                _sizeOfImportAddressTable + 
+            return
+                _sizeOfImportAddressTable +
                 72 + // size of CLR header
-                + BitArithmeticUtilities.Align(ilStreamLength, 4);
+                BitArithmeticUtilities.Align(ilStreamLength, 4);
         }
 
         private const int ImageDebugDirectoryBaseSize =

src/Compilers/VisualBasic/Test/Emit/Emit/DeterministicTests.vb

-
-' Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+' Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
 Imports System
 Imports System.Collections.Generic
@@ -19,8 +18,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.UnitTests.Emit
 
             Dim compilation = CreateCompilationWithMscorlib({source}, assemblyName:="DeterminismTest", options:=options)
 
-            ' The resolution of the PE header time date stamp Is seconds, And we want to make sure that has an opportunity to change
-            ' between calls to Emit.
+            ' The resolution of the PE header time date stamp is seconds, and we want to make sure
+            ' that has an opportunity to change between calls to Emit.
             Thread.Sleep(TimeSpan.FromSeconds(1))
 
             Return compilation.EmitToArray()