Merge pull request #2137 from tmat/PdbNativeStream

Use an unmanaged stream during PDB writing

src/Compilers/CSharp/Test/Emit/Emit/CompilationEmitTests.cs

@@ -2726,7 +2726,8 @@ public void BrokenPDBStream()
 
             Assert.Equal((int)ErrorCode.FTL_DebugEmitFailure, err.Code);
             Assert.Equal(1, err.Arguments.Count);
-            Assert.True(((string)err.Arguments[0]).EndsWith(" HRESULT: 0x806D0004", StringComparison.Ordinal));
+            var ioExceptionMessage = new IOException().Message;
+            Assert.Equal(ioExceptionMessage, (string)err.Arguments[0]);
 
             pdb.Dispose();
             result = compilation.Emit(output, pdb);
@@ -2736,7 +2737,7 @@ public void BrokenPDBStream()
 
             Assert.Equal((int)ErrorCode.FTL_DebugEmitFailure, err.Code);
             Assert.Equal(1, err.Arguments.Count);
-            Assert.True(((string)err.Arguments[0]).EndsWith(" HRESULT: 0x806D0004", StringComparison.Ordinal));
+            Assert.Equal(ioExceptionMessage, (string)err.Arguments[0]);
         }
 
         [Fact]

src/Compilers/Core/Portable/Compilation/Compilation.cs

@@ -1624,7 +1624,6 @@ private static EmitResult ToEmitResultAndFree(DiagnosticBag diagnostics, bool su
             Stream signingInputStream = null;
             DiagnosticBag metadataDiagnostics = null;
             DiagnosticBag pdbBag = null;
-            Stream pdbTempStream = null;
             Stream peStream = null;
             Stream peTempStream = null;
 
@@ -1637,21 +1636,10 @@ private static EmitResult ToEmitResultAndFree(DiagnosticBag diagnostics, bool su
 
                 if (pdbStreamProvider != null)
                 {
-                    var nativePdbStream = pdbStreamProvider.Stream;
-
-                    // Native PDB writer is able to update an existing stream.
-                    // It checks for length to determine whether the given stream has existing data to be updated,
-                    // or whether it should start writing PDB data from scratch. Thus if not writing to a seekable empty stream,
-                    // we have to create an in-memory temp stream for the PDB writer and copy all data to the actual stream at once at the end.
-                    if (nativePdbStream == null || !nativePdbStream.CanSeek || nativePdbStream.Length != 0)
-                    {
-                        nativePdbStream = pdbTempStream = new MemoryStream();
-                    }
-
                     // The calls ISymUnmanagedWriter2.GetDebugInfo require a file name in order to succeed.  This is 
                     // frequently used during PDB writing.  Ensure a name is provided here in the case we were given
                     // only a Stream value.
-                    nativePdbWriter = new Cci.PdbWriter(nativePdbStream, pdbPath, testSymWriterFactory);
+                    nativePdbWriter = new Cci.PdbWriter(pdbPath, testSymWriterFactory);
                 }
 
                 Func<Stream> getPeStream = () =>
@@ -1715,20 +1703,14 @@ private static EmitResult ToEmitResultAndFree(DiagnosticBag diagnostics, bool su
                             peTempStream.CopyTo(peStream);
                         }
 
-                        if (pdbTempStream != null)
+                        if (nativePdbWriter != null)
                         {
-                            // Note: Native PDB writer may operate on the underlying stream during disposal.
-                            // So close it here before we read data from the underlying stream.
-                            nativePdbWriter?.Dispose();
-                            nativePdbWriter = null;
-
                             var pdbStream = pdbStreamProvider.GetOrCreateStream(metadataDiagnostics);
                             Debug.Assert(pdbStream != null || metadataDiagnostics.HasAnyErrors());
 
                             if (pdbStream != null)
                             {
-                                pdbTempStream.Position = 0;
-                                pdbTempStream.CopyTo(pdbStream);
+                                nativePdbWriter.WriteTo(pdbStream);
                             }
                         }
                     }
@@ -1774,7 +1756,6 @@ private static EmitResult ToEmitResultAndFree(DiagnosticBag diagnostics, bool su
             {
                 nativePdbWriter?.Dispose();
                 peTempStream?.Dispose();
-                pdbTempStream?.Dispose();
                 signingInputStream?.Dispose();
                 pdbBag?.Free();
                 metadataDiagnostics?.Free();
@@ -1797,7 +1778,6 @@ private static EmitResult ToEmitResultAndFree(DiagnosticBag diagnostics, bool su
             CancellationToken cancellationToken)
         {
             using (var pdbWriter = new Cci.PdbWriter(
-                pdbStream,
                 moduleBeingBuilt.EmitOptions.PdbFilePath ?? FileNameUtilities.ChangeExtension(SourceModule.Name, "pdb"),
                 testSymWriterFactory))
             {
@@ -1819,6 +1799,8 @@ private static EmitResult ToEmitResultAndFree(DiagnosticBag diagnostics, bool su
                     writer.WriteMetadataAndIL(pdbWriter, metadataStream, ilStream, out metadataSizes);
                     writer.GetMethodTokens(updatedMethods);
 
+                    pdbWriter.WriteTo(pdbStream);
+
                     return diagnostics.HasAnyErrors() ? null : writer.GetDelta(baseline, this, encId, metadataSizes);
                 }
                 catch (Cci.PdbWritingException e)

src/Compilers/Core/Portable/PEWriter/PdbWriter.cs

@@ -7,6 +7,7 @@
 using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
+using System.Runtime.InteropServices.ComTypes;
 using System.Text;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Collections;
@@ -36,9 +37,9 @@ internal sealed class PdbWriter : IDisposable
 
         private static Type s_lazyCorSymWriterSxSType;
 
-        private readonly Stream _stream;
         private readonly string _fileName;
         private readonly Func<object> _symWriterFactory;
+        private IStream _nativeStream;
         private MetadataWriter _metadataWriter;
         private ISymUnmanagedWriter2 _symWriter;
 
@@ -54,15 +55,61 @@ internal sealed class PdbWriter : IDisposable
         private uint[] _sequencePointEndLines;
         private uint[] _sequencePointEndColumns;
 
-        public PdbWriter(Stream stream, string fileName, Func<object> symWriterFactory = null)
+        [DllImport("shlwapi.dll", ExactSpelling = true, CharSet = CharSet.Unicode)]
+        private extern static IStream SHCreateMemStream([In] IntPtr pInit, [In] uint cbInit);
+
+        public PdbWriter(string fileName, Func<object> symWriterFactory = null)
         {
-            Debug.Assert(stream != null);
-            _stream = stream;
             _fileName = fileName;
             _symWriterFactory = symWriterFactory;
             CreateSequencePointBuffers(capacity: 64);
         }
 
+        public unsafe void WriteTo(Stream stream)
+        {
+            Debug.Assert(_nativeStream != null);
+            Debug.Assert(_symWriter != null);
+
+            try
+            {
+                // SymWriter flushes data to the native stream on close:
+                _symWriter.Close();
+                _symWriter = null;
+
+                var statStg = default(STATSTG);
+                _nativeStream.Stat(out statStg, grfStatFlag: 3 /*STATFLAG_NONAME | STATFLAG_NOOPEN*/);
+                _nativeStream.Seek(0L, 0, IntPtr.Zero);
+
+                int size = (int)statStg.cbSize;
+
+                // If the target stream allows seeking set its length upfront.
+                // When writing to a large file, it helps to give a hint to the OS how big the file is going to be.
+                if (stream.CanSeek)
+                {
+                    stream.SetLength(stream.Position + size);
+                }
+
+                // Copy unmanagedStream contents to stream in chunks
+                var buffer = new byte[Math.Min(size, 4096)];
+                while (true)
+                {
+                    int bytesRead = 0;
+                    _nativeStream.Read(buffer, buffer.Length, (IntPtr)(&bytesRead));
+
+                    if (bytesRead <= 0)
+                    {
+                        break;
+                    }
+
+                    stream.Write(buffer, 0, bytesRead);
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new PdbWritingException(ex);
+            }
+        }
+
         public void Dispose()
         {
             Close();
@@ -73,16 +120,19 @@ public void Dispose()
         {
             Close();
         }
-
-        /// <summary>
-        /// Close the PDB writer and write the PDB data to <see cref="_stream"/>.
-        /// </summary>
+        
         private void Close()
         {
             try
             {
                 _symWriter?.Close();
                 _symWriter = null;
+
+                if (_nativeStream != null)
+                {
+                    Marshal.ReleaseComObject(_nativeStream);
+                    _nativeStream = null;
+                }
             }
             catch (Exception ex)
             {
@@ -546,11 +596,13 @@ public void SetMetadataEmitter(MetadataWriter metadataWriter)
             {
                 var instance = (ISymUnmanagedWriter2)(_symWriterFactory != null ? _symWriterFactory() : Activator.CreateInstance(GetCorSymWriterSxSType()));
 
-                // Important: If the stream is not specified or if it is non-empty the SymWriter appends data to it (provided it contains valid PDB)
+                // Correctness: If the stream is not specified or if it is non-empty the SymWriter appends data to it (provided it contains valid PDB)
                 // and the resulting PDB has Age = existing_age + 1.
-                Debug.Assert(_stream.Length == 0);
+                // PERF: Use a native stream for the write and copy it at the end. This reduces the total GC
+                // allocations for the COM interop and geometric growth of the underlying managed stream.
+                _nativeStream = SHCreateMemStream(IntPtr.Zero, 0u);
 
-                instance.Initialize(new PdbMetadataWrapper(metadataWriter), _fileName, new ComStreamWrapper(_stream), fullBuild: true);
+                instance.Initialize(new PdbMetadataWrapper(metadataWriter), _fileName, _nativeStream, fullBuild: true);
 
                 _metadataWriter = metadataWriter;
                 _symWriter = instance;