From c94b9fa839cd1ff193bc4af7d87d6b14f6bff3b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Mon, 8 Jun 2020 18:05:20 -0700 Subject: [PATCH] Compiler Options in PDB fixups (#44950) * Compiler Options in PDB fixups * Use MetadataVisualizerOptions.NoHeapReferences * Add doc back --- ...able-pdb.md => pdb-compilation-options.md} | 25 +++++++------ ...SharpDeterministicBuildCompilationTests.cs | 2 +- .../CSharp/Test/Emit/PDB/PDBAsyncTests.cs | 22 +++++------ .../CSharp/Test/Emit/PDB/PDBIteratorTests.cs | 22 +++++------ .../CSharp/Test/Emit/PDB/PDBUsingTests.cs | 37 +++++++++---------- .../PEWriter/MetadataWriter.PortablePdb.cs | 2 +- .../Test/Emit/PDB/PDBAsyncTests.vb | 6 +-- .../Test/Emit/PDB/PDBIteratorTests.vb | 6 +-- ...BasicDeterministicBuildCompilationTests.vb | 2 +- .../PortableCustomDebugInfoKinds.cs | 2 +- 10 files changed, 64 insertions(+), 62 deletions(-) rename docs/features/{compilation-from-portable-pdb.md => pdb-compilation-options.md} (92%) diff --git a/docs/features/compilation-from-portable-pdb.md b/docs/features/pdb-compilation-options.md similarity index 92% rename from docs/features/compilation-from-portable-pdb.md rename to docs/features/pdb-compilation-options.md index 3b0a863a21f..88a6a27325b 100644 --- a/docs/features/compilation-from-portable-pdb.md +++ b/docs/features/pdb-compilation-options.md @@ -1,19 +1,22 @@ -## compilation-from-portable-pdb +# Embedding Compilation options in Portable PDBs -Compilation from portable PDBs today is not completely possible, but is desirable in order to help reconstruct a compilation from source provided via source link or embedded in a pdb. Motivation is derived from [roslyn 41395](https://github.com/dotnet/roslyn/issues/41395). Once work is finalized on this, a compilation should be able to be made that is exactly the same as an initial compilation as long as it meets the conditions outlined in the assumptions below. [roslyn#44703](https://github.com/dotnet/roslyn/issues/44703) tracks adding an end-to-end validation of the scenario and should also update documentation here to further outline the steps. +Prior to this feature the compiler did not emit all information that's necessary to reconstruct the original compilation to the output binaries. +It is desirable to include such information in order to support scenarios such as validation that given binaries can be reproduced from their original sources, [post-build source analysis](https://github.com/dotnet/roslyn/issues/41395), etc. + +The goal of this feature is to be able to construct a compilation that is exactly the same as an initial compilation as long as it meets the conditions outlined in the assumptions below. This document is restricted to the following assumptions: -1. The benefit is for builds with `-deterministic` and published to the symbol server. -2. Source generator and analyzer references are not needed for this task. They may be useful, but are out of scope for this document. +1. The full benefit is for builds with `-deterministic` and published to the symbol server. That said the compiler embeds compilation options and references to all Portable PDBs. +2. Source generator and analyzer references are not needed for this task. They may be useful, but are out of scope for this feature. 3. Any storage capacity used for PDBs and source should not impact this feature, such as compression algorithm. 4. Only Portable PDB files will be included for this spec. This feature can be expanded past these once it is implemented and proven needed elsewhere. -This document will provide the expanded specification to the Portable PDB format. Any additions to that format will be ported to expand documentation provided in [dotnet-runtime](https://github.com/jnm2/dotnet-runtime/blob/26efe3467741fe2a85780b2d2cd18875af6ebd98/docs/design/specs/PortablePdb-Metadata.md#source-link-c-and-vb-compilers). +This document will provide the expanded specification to the Portable PDB format. Any additions to that format will be ported to expand documentation provided in [dotnet-runtime](https://github.com/dotnet/runtime/blob/master/docs/design/specs/PortablePdb-Metadata.md). ## PDB Format Additions -#### Metadata References +#### Compilation Metadata References custom debug information Symbol server uses a [key](https://github.com/dotnet/symstore/blob/master/docs/specs/SSQP_Key_Conventions.md#pe-timestamp-filesize) computed from the COFF header in the PE image: @@ -22,13 +25,13 @@ Size of image: 4 byte integer Example: -File name: `example.exe` + File name: `example.exe` -COFF header Timestamp field: `0x542d5742` + COFF header Timestamp field: `0x542d5742` -COFF header SizeOfImage field: `0x32000` + COFF header SizeOfImage field: `0x32000` -Lookup key: `example.exe/542d574232000/example.exe` + Lookup key: `example.exe/542d574232000/example.exe` To fully support metadata references, a user will need to be able to find the exact PE image that was used in the compilation. This will be done by storing the parts that make up the symbol server key. The MVID of a reference will be stored since it's a GUID that represents the symbol. This is to future proof the information for reference lookup. @@ -128,7 +131,7 @@ foreach (var handle in metadataReader.GetCustomDebugInformation(EntityHandle.Mod } ``` -### Compiler Flag Key Value Pairs +### Compiler Options custom debug information The remaining values will be stored as key value pairs in the pdb. The storage format will be UTF8 encoded key value pairs that are null terminated. Order is not guaranteed. Any values left out can be assumed to be the default for the type. Keys may be different for Visual Basic and CSharp. They are serialized to reflect the command line arguments representing the same values diff --git a/src/Compilers/CSharp/Test/Emit/PDB/CSharpDeterministicBuildCompilationTests.cs b/src/Compilers/CSharp/Test/Emit/PDB/CSharpDeterministicBuildCompilationTests.cs index 2400fa12a0e..b335137c089 100644 --- a/src/Compilers/CSharp/Test/Emit/PDB/CSharpDeterministicBuildCompilationTests.cs +++ b/src/Compilers/CSharp/Test/Emit/PDB/CSharpDeterministicBuildCompilationTests.cs @@ -77,7 +77,7 @@ private static void TestDeterministicCompilationCSharp(string langVersion, Synta { var pdbReader = embeddedPdb.GetMetadataReader(); - var metadataReferenceReader = DeterministicBuildCompilationTestHelpers.GetSingleBlob(PortableCustomDebugInfoKinds.MetadataReferenceInfo, pdbReader); + var metadataReferenceReader = DeterministicBuildCompilationTestHelpers.GetSingleBlob(PortableCustomDebugInfoKinds.CompilationMetadataReferences, pdbReader); var compilationOptionsReader = DeterministicBuildCompilationTestHelpers.GetSingleBlob(PortableCustomDebugInfoKinds.CompilationOptions, pdbReader); VerifyCompilationOptions(compilationOptions, originalCompilation, emitOptions, compilationOptionsReader, langVersion); diff --git a/src/Compilers/CSharp/Test/Emit/PDB/PDBAsyncTests.cs b/src/Compilers/CSharp/Test/Emit/PDB/PDBAsyncTests.cs index f8e6d4a63ec..70c98bb13aa 100644 --- a/src/Compilers/CSharp/Test/Emit/PDB/PDBAsyncTests.cs +++ b/src/Compilers/CSharp/Test/Emit/PDB/PDBAsyncTests.cs @@ -1968,8 +1968,8 @@ static async Task G() "); } + [Fact] [WorkItem(17934, "https://github.com/dotnet/roslyn/issues/17934")] - [ConditionalFact(AlwaysSkip = "https://github.com/dotnet/roslyn/issues/44901")] public void PartialKickoffMethod() { string src = @" @@ -2002,22 +2002,23 @@ public partial class C pdbStream, options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb)); + Assert.True(result.Success); pdbStream.Position = 0; - using (var provider = MetadataReaderProvider.FromPortablePdbStream(pdbStream)) - { - var mdReader = provider.GetMetadataReader(); - var writer = new StringWriter(); - var visualizer = new MetadataVisualizer(mdReader, writer); - visualizer.WriteMethodDebugInformation(); - AssertEx.AssertEqualToleratingWhitespaceDifferences(@" + using var provider = MetadataReaderProvider.FromPortablePdbStream(pdbStream); + var mdReader = provider.GetMetadataReader(); + var writer = new StringWriter(); + var visualizer = new MetadataVisualizer(mdReader, writer, MetadataVisualizerOptions.NoHeapReferences); + visualizer.WriteMethodDebugInformation(); + + AssertEx.AssertEqualToleratingWhitespaceDifferences(@" MethodDebugInformation (index: 0x31, size: 20): ================================================== 1: nil 2: nil 3: nil -4: #e5 +4: { Kickoff Method: 0x06000001 (MethodDef) Locals: 0x11000002 (StandAloneSig) @@ -2029,8 +2030,7 @@ public partial class C IL_002A: } 5: nil", - writer.ToString()); - } + writer.ToString()); } [Fact] diff --git a/src/Compilers/CSharp/Test/Emit/PDB/PDBIteratorTests.cs b/src/Compilers/CSharp/Test/Emit/PDB/PDBIteratorTests.cs index fd5ecb316b5..39a3b9808c4 100644 --- a/src/Compilers/CSharp/Test/Emit/PDB/PDBIteratorTests.cs +++ b/src/Compilers/CSharp/Test/Emit/PDB/PDBIteratorTests.cs @@ -1562,8 +1562,8 @@ static IEnumerable F() "); } + [Fact] [WorkItem(8473, "https://github.com/dotnet/roslyn/issues/8473")] - [ConditionalFact(AlwaysSkip = "https://github.com/dotnet/roslyn/issues/44901")] public void PortableStateMachineDebugInfo() { string src = @" @@ -1586,22 +1586,23 @@ public class C pdbStream, options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb)); + Assert.True(result.Success); pdbStream.Position = 0; - using (var provider = MetadataReaderProvider.FromPortablePdbStream(pdbStream)) - { - var mdReader = provider.GetMetadataReader(); - var writer = new StringWriter(); - var visualizer = new MetadataVisualizer(mdReader, writer); - visualizer.WriteMethodDebugInformation(); - AssertEx.AssertEqualToleratingWhitespaceDifferences(@" + using var provider = MetadataReaderProvider.FromPortablePdbStream(pdbStream); + var mdReader = provider.GetMetadataReader(); + var writer = new StringWriter(); + var visualizer = new MetadataVisualizer(mdReader, writer, MetadataVisualizerOptions.NoHeapReferences); + visualizer.WriteMethodDebugInformation(); + + AssertEx.AssertEqualToleratingWhitespaceDifferences(@" MethodDebugInformation (index: 0x31, size: 40): ================================================== 1: nil 2: nil 3: nil 4: nil -5: #d0 +5: { Kickoff Method: 0x06000001 (MethodDef) Locals: 0x11000001 (StandAloneSig) @@ -1618,8 +1619,7 @@ public class C 9: nil a: nil ", - writer.ToString()); - } + writer.ToString()); } } } diff --git a/src/Compilers/CSharp/Test/Emit/PDB/PDBUsingTests.cs b/src/Compilers/CSharp/Test/Emit/PDB/PDBUsingTests.cs index bc0d1211852..092b6f60172 100644 --- a/src/Compilers/CSharp/Test/Emit/PDB/PDBUsingTests.cs +++ b/src/Compilers/CSharp/Test/Emit/PDB/PDBUsingTests.cs @@ -2474,7 +2474,7 @@ void M() "); } - [ConditionalFact(AlwaysSkip = "https://github.com/dotnet/roslyn/issues/44901")] + [Fact] public void ImportScopeEquality() { var sources = new[] { @" @@ -2535,29 +2535,28 @@ class C6 { void F() {} } var pdbStream = new MemoryStream(); c.EmitToArray(EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), pdbStream: pdbStream); var pdbImage = pdbStream.ToImmutable(); - using (var metadata = new PinnedMetadata(pdbImage)) - { - var mdReader = metadata.Reader; - var writer = new StringWriter(); - var mdVisualizer = new MetadataVisualizer(mdReader, writer); - mdVisualizer.WriteImportScope(); - AssertEx.AssertEqualToleratingWhitespaceDifferences(@" + using var metadata = new PinnedMetadata(pdbImage); + var mdReader = metadata.Reader; + var writer = new StringWriter(); + var mdVisualizer = new MetadataVisualizer(mdReader, writer, MetadataVisualizerOptions.NoHeapReferences); + mdVisualizer.WriteImportScope(); + + AssertEx.AssertEqualToleratingWhitespaceDifferences(@" ImportScope (index: 0x35, size: 36): -================================================================================================ - Parent Imports -================================================================================================ -1: nil (ImportScope) 'A' (#1) = 0x23000002 (AssemblyRef) -2: 0x35000001 (ImportScope) Extern Alias 'A' (#1), 'System' (#b6) -3: 0x35000001 (ImportScope) Extern Alias 'A' (#1), 'System' (#b6), 'C' (#cd) = 'System' (#b6) +======================================================================== + Parent Imports +======================================================================== +1: nil (ImportScope) 'A' = 0x23000002 (AssemblyRef) +2: 0x35000001 (ImportScope) Extern Alias 'A', 'System' +3: 0x35000001 (ImportScope) Extern Alias 'A', 'System', 'C' = 'System' 4: 0x35000003 (ImportScope) nil -5: 0x35000004 (ImportScope) 'System.Collections' (#da) -6: 0x35000004 (ImportScope) 'System.Collections.Generic' (#ff) -7: 0x35000001 (ImportScope) Extern Alias 'A' (#1), 'System' (#b6), 'D' (#11e) = 'System' (#b6) +5: 0x35000004 (ImportScope) 'System.Collections' +6: 0x35000004 (ImportScope) 'System.Collections.Generic' +7: 0x35000001 (ImportScope) Extern Alias 'A', 'System', 'D' = 'System' 8: 0x35000007 (ImportScope) nil -9: 0x35000008 (ImportScope) 'System.Collections' (#da) +9: 0x35000008 (ImportScope) 'System.Collections' ", writer.ToString()); - } } } } diff --git a/src/Compilers/Core/Portable/PEWriter/MetadataWriter.PortablePdb.cs b/src/Compilers/Core/Portable/PEWriter/MetadataWriter.PortablePdb.cs index d0174644ef9..b18e22f0582 100644 --- a/src/Compilers/Core/Portable/PEWriter/MetadataWriter.PortablePdb.cs +++ b/src/Compilers/Core/Portable/PEWriter/MetadataWriter.PortablePdb.cs @@ -950,7 +950,7 @@ private void EmbedMetadataReferenceInformation(CommonPEModuleBuilder module) _debugMetadataOpt.AddCustomDebugInformation( parent: EntityHandle.ModuleDefinition, - kind: _debugMetadataOpt.GetOrAddGuid(PortableCustomDebugInfoKinds.MetadataReferenceInfo), + kind: _debugMetadataOpt.GetOrAddGuid(PortableCustomDebugInfoKinds.CompilationMetadataReferences), value: _debugMetadataOpt.GetOrAddBlob(builder)); static PEReader GetReader(ISymbol symbol) diff --git a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBAsyncTests.vb b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBAsyncTests.vb index 156d9748fdc..bf0f3a18452 100644 --- a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBAsyncTests.vb +++ b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBAsyncTests.vb @@ -898,7 +898,7 @@ End Module ", sequencePoints:="M+VB$StateMachine_0_F.MoveNext") End Sub - + Public Sub PartialKickoffMethod() Dim src = " Public Partial Class C @@ -934,7 +934,7 @@ End Class Using provider = MetadataReaderProvider.FromPortablePdbStream(pdbStream) Dim mdReader = provider.GetMetadataReader() Dim writer = New StringWriter() - Dim visualizer = New MetadataVisualizer(mdReader, writer) + Dim visualizer = New MetadataVisualizer(mdReader, writer, MetadataVisualizerOptions.NoHeapReferences) visualizer.WriteMethodDebugInformation() AssertEx.AssertEqualToleratingWhitespaceDifferences(" @@ -943,7 +943,7 @@ MethodDebugInformation (index: 0x31, size: 20): 1: nil 2: nil 3: nil -4: #125 +4: { Kickoff Method: 0x06000002 (MethodDef) Locals: 0x11000002 (StandAloneSig) diff --git a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBIteratorTests.vb b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBIteratorTests.vb index 45cde7f4eed..517a930aaf4 100644 --- a/src/Compilers/VisualBasic/Test/Emit/PDB/PDBIteratorTests.vb +++ b/src/Compilers/VisualBasic/Test/Emit/PDB/PDBIteratorTests.vb @@ -530,8 +530,8 @@ End Module ) End Sub + - Public Sub PortableStateMachineDebugInfo() Dim src = " Imports System.Collections.Generic @@ -551,7 +551,7 @@ End Class" Using provider = MetadataReaderProvider.FromPortablePdbStream(pdbStream) Dim mdReader = provider.GetMetadataReader() Dim writer = New StringWriter() - Dim visualizer = New MetadataVisualizer(mdReader, writer) + Dim visualizer = New MetadataVisualizer(mdReader, writer, MetadataVisualizerOptions.NoHeapReferences) visualizer.WriteMethodDebugInformation() AssertEx.AssertEqualToleratingWhitespaceDifferences(" @@ -561,7 +561,7 @@ MethodDebugInformation (index: 0x31, size: 40): 2: nil 3: nil 4: nil -5: #10e +5: { Kickoff Method: 0x06000002 (MethodDef) Locals: 0x11000002 (StandAloneSig) diff --git a/src/Compilers/VisualBasic/Test/Emit/PDB/VisualBasicDeterministicBuildCompilationTests.vb b/src/Compilers/VisualBasic/Test/Emit/PDB/VisualBasicDeterministicBuildCompilationTests.vb index ab8d4ee010d..8b37ea1abfe 100644 --- a/src/Compilers/VisualBasic/Test/Emit/PDB/VisualBasicDeterministicBuildCompilationTests.vb +++ b/src/Compilers/VisualBasic/Test/Emit/PDB/VisualBasicDeterministicBuildCompilationTests.vb @@ -63,7 +63,7 @@ Public Class VisualBasicDeterministicBuildCompilationTests Using embeddedPdb As MetadataReaderProvider = peReader.ReadEmbeddedPortablePdbDebugDirectoryData(embedded) Dim pdbReader = embeddedPdb.GetMetadataReader() - Dim metadataReferenceReader = DeterministicBuildCompilationTestHelpers.GetSingleBlob(PortableCustomDebugInfoKinds.MetadataReferenceInfo, pdbReader) + Dim metadataReferenceReader = DeterministicBuildCompilationTestHelpers.GetSingleBlob(PortableCustomDebugInfoKinds.CompilationMetadataReferences, pdbReader) Dim compilationOptionsReader = DeterministicBuildCompilationTestHelpers.GetSingleBlob(PortableCustomDebugInfoKinds.CompilationOptions, pdbReader) VerifyCompilationOptions(compilationOptions, compilationOptionsReader, emitOptions, originalCompilation) diff --git a/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs b/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs index bfd14bcf7fa..9dd256928dd 100644 --- a/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs +++ b/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs @@ -17,7 +17,7 @@ internal static class PortableCustomDebugInfoKinds public static readonly Guid EncLambdaAndClosureMap = new Guid("A643004C-0240-496F-A783-30D64F4979DE"); public static readonly Guid SourceLink = new Guid("CC110556-A091-4D38-9FEC-25AB9A351A6A"); public static readonly Guid EmbeddedSource = new Guid("0E8A571B-6926-466E-B4AD-8AB04611F5FE"); - public static readonly Guid MetadataReferenceInfo = new Guid("7E4D4708-096E-4C5C-AEDA-CB10BA6A740D"); + public static readonly Guid CompilationMetadataReferences = new Guid("7E4D4708-096E-4C5C-AEDA-CB10BA6A740D"); public static readonly Guid CompilationOptions = new Guid("B5FEEC05-8CD0-4A83-96DA-466284BB4BD8"); } } -- GitLab