Signed integer compression in PE Writer was implemented incorrectly. Fix the...

Signed integer compression in PE Writer was implemented incorrectly. Fix the implementation. This didn't actually affect anything Roslyn currently emits since we are only using this compression to encode low bounds of arrays, which are always 0 in C# and VB.
     Support arbitrary negative syntax offsets in EditAndContinueMethodDebugInformation. Rather than using signed integers to encode the negative values, which increases the size of the data for the majority of cases when the offsets are positive, we encode a baseline and add it to the offsets. (changeset 1365821)

Src/Compilers/Core/CodeAnalysisTest/Emit/CustomDebugInfoReaderTests.cs

 // Copyright (c) Microsoft Open Technologies, Inc.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 extern alias PDB;
+
+using System.Collections.Immutable;
+using Microsoft.CodeAnalysis.CodeGen;
+using Microsoft.CodeAnalysis.Emit;
 using PDB::Roslyn.Utilities.Pdb;
 using Roslyn.Test.Utilities;
 using Xunit;
@@ -133,5 +137,25 @@ public void TryGetCustomDebugInfoRecord1()
 
             AssertEx.Equal(new byte[] { 0xab }, CustomDebugInfoReader.TryGetCustomDebugInfoRecord(cdi, CustomDebugInfoKind.DynamicLocals));
         }
+
+        [Fact]
+        public void EditAndContinueLocalSlotMap_NegativeSyntaxOffsets()
+        {
+            var slots = ImmutableArray.Create(
+                new LocalSlotDebugInfo(SynthesizedLocalKind.UserDefined, new LocalDebugId(-1, 10)),
+                new LocalSlotDebugInfo(SynthesizedLocalKind.TryAwaitPendingCaughtException, new LocalDebugId(-20000, 10)));
+
+            var customMetadata = new Cci.MemoryStream();
+            var cmw = new Cci.BinaryWriter(customMetadata);
+
+            new EditAndContinueMethodDebugInformation(slots).SerializeLocalSlots(cmw);
+
+            var bytes = customMetadata.ToImmutableArray();
+            AssertEx.Equal(new byte[] { 0xFE, 0xC0, 0x00, 0x4E, 0x20, 0x81, 0xC0, 0x00, 0x4E, 0x1F, 0x0A, 0x9A, 0x00, 0x0A }, bytes);
+
+            var deserialized = EditAndContinueMethodDebugInformation.Create(bytes).LocalSlots;
+
+            AssertEx.Equal(slots, deserialized);
+        }
     }
 }

Src/Compilers/Core/Portable/CodeGen/LocalSlotDebugInfo.cs

 // Copyright (c) Microsoft Open Technologies, Inc.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
+using System;
+using Roslyn.Utilities;
+
 namespace Microsoft.CodeAnalysis.CodeGen
 {
-    internal struct LocalSlotDebugInfo
+    internal struct LocalSlotDebugInfo : IEquatable<LocalSlotDebugInfo>
     {
         public readonly SynthesizedLocalKind SynthesizedKind;
         public readonly LocalDebugId Id;
@@ -12,5 +15,21 @@ public LocalSlotDebugInfo(SynthesizedLocalKind synthesizedKind, LocalDebugId id)
             this.SynthesizedKind = synthesizedKind;
             this.Id = id;
         }
+
+        public bool Equals(LocalSlotDebugInfo other)
+        {
+            return this.SynthesizedKind == other.SynthesizedKind
+                && this.Id.Equals(other.Id);
+        }
+
+        public override bool Equals(object obj)
+        {
+            return obj is LocalSlotDebugInfo && Equals((LocalSlotDebugInfo)obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return Hash.Combine((int)SynthesizedKind, Id.GetHashCode());
+        }
     }
 }

Src/Compilers/Core/Portable/Emit/EditAndContinueMethodDebugInformation.cs

@@ -29,6 +29,7 @@ public static EditAndContinueMethodDebugInformation Create(ImmutableArray<byte>
         }
 
         private const byte AlignmentValue = 0xff;
+        private const byte SyntaxOffsetBaseline = 0xfe;
 
         private unsafe static ImmutableArray<LocalSlotDebugInfo> UncompressSlotMap(ImmutableArray<byte> compressedSlotMap)
         {
@@ -38,11 +39,11 @@ private unsafe static ImmutableArray<LocalSlotDebugInfo> UncompressSlotMap(Immut
             }
 
             var mapBuilder = ArrayBuilder<LocalSlotDebugInfo>.GetInstance();
-            
+            int syntaxOffsetBaseline = -1;
+
             fixed (byte* compressedSlotMapPtr = &compressedSlotMap.ToArray()[0])
             {
                 var blobReader = new BlobReader(compressedSlotMapPtr, compressedSlotMap.Length);
-
                 while (blobReader.RemainingBytes > 0)
                 {
                     byte b = blobReader.ReadByte();
@@ -52,6 +53,12 @@ private unsafe static ImmutableArray<LocalSlotDebugInfo> UncompressSlotMap(Immut
                         break;
                     }
 
+                    if (b == SyntaxOffsetBaseline)
+                    {
+                        syntaxOffsetBaseline = -blobReader.ReadCompressedInteger();
+                        continue;
+                    }
+
                     if (b == 0)
                     {
                         // short-lived temp, no info
@@ -62,14 +69,13 @@ private unsafe static ImmutableArray<LocalSlotDebugInfo> UncompressSlotMap(Immut
                     var kind = (SynthesizedLocalKind)((b & 0x3f) - 1);
                     bool hasOrdinal = (b & (1 << 7)) != 0;
 
-                    // TODO: Right now all integers are >= -1, but we should not assume that and read Ecma335 compressed int instead.
                     int syntaxOffset;
                     if (!blobReader.TryReadCompressedInteger(out syntaxOffset)) 
                     {
                         return default(ImmutableArray<LocalSlotDebugInfo>);
                     }
 
-                    syntaxOffset--;
+                    syntaxOffset += syntaxOffsetBaseline;
 
                     int ordinal = 0;
                     if (hasOrdinal && !blobReader.TryReadCompressedInteger(out ordinal))
@@ -92,7 +98,7 @@ internal void SerializeCustomDebugInformation(ArrayBuilder<Cci.MemoryStream> cus
             }
 
             Cci.MemoryStream customMetadata = new Cci.MemoryStream();
-            Cci.BinaryWriter cmw = new Cci.BinaryWriter(customMetadata, true);
+            Cci.BinaryWriter cmw = new Cci.BinaryWriter(customMetadata);
             cmw.WriteByte(4); // version
             cmw.WriteByte(6); // kind: EditAndContinueLocalSlotMap
             cmw.Align(4);
@@ -101,6 +107,41 @@ internal void SerializeCustomDebugInformation(ArrayBuilder<Cci.MemoryStream> cus
             uint lengthPosition = cmw.BaseStream.Position;
             cmw.WriteUint(0);
 
+            SerializeLocalSlots(cmw);
+
+            uint length = customMetadata.Position;
+
+            // align with values that the reader skips
+            while (length % 4 != 0)
+            {
+                cmw.WriteByte(AlignmentValue);
+                length++;
+            }
+
+            cmw.BaseStream.Position = lengthPosition;
+            cmw.WriteUint(length);
+            cmw.BaseStream.Position = length;
+
+            customDebugInfo.Add(customMetadata);
+        }
+
+        internal void SerializeLocalSlots(Cci.BinaryWriter cmw)
+        {
+            int syntaxOffsetBaseline = -1;
+            foreach (LocalSlotDebugInfo localSlot in this.LocalSlots)
+            {
+                if (localSlot.Id.SyntaxOffset < syntaxOffsetBaseline)
+                {
+                    syntaxOffsetBaseline = localSlot.Id.SyntaxOffset;
+                }
+            }
+
+            if (syntaxOffsetBaseline != -1)
+            {
+                cmw.WriteByte(SyntaxOffsetBaseline);
+                cmw.WriteCompressedUInt((uint)(-syntaxOffsetBaseline));
+            }
+
             foreach (LocalSlotDebugInfo localSlot in this.LocalSlots)
             {
                 var kind = localSlot.SynthesizedKind;
@@ -121,30 +162,13 @@ internal void SerializeCustomDebugInformation(ArrayBuilder<Cci.MemoryStream> cus
                 }
 
                 cmw.WriteByte(b);
-
-                // TODO: Right now all integers are >= -1, but we should not assume that and write Ecma335 compressed int instead.
-                cmw.WriteCompressedUInt(unchecked((uint)(localSlot.Id.SyntaxOffset + 1)));
+                cmw.WriteCompressedUInt((uint)(localSlot.Id.SyntaxOffset - syntaxOffsetBaseline));
 
                 if (hasOrdinal)
                 {
                     cmw.WriteCompressedUInt((uint)localSlot.Id.Ordinal);
                 }
             }
-
-            uint length = customMetadata.Position;
-
-            // align with values that the reader skips
-            while (length % 4 != 0)
-            {
-                cmw.WriteByte(AlignmentValue);
-                length++;
-            }
-
-            cmw.BaseStream.Position = lengthPosition;
-            cmw.WriteUint(length);
-            cmw.BaseStream.Position = length;
-
-            customDebugInfo.Add(customMetadata);
         }
     }
 }

Src/Compilers/Core/Portable/PEWriter/BinaryWriter.cs

@@ -380,43 +380,36 @@ internal void WriteString(string str, bool emitNullTerminator)
             }
         }
 
-        internal void WriteCompressedInt(int val)
+        internal void WriteCompressedSignedInteger(int value)
         {
             unchecked
             {
-                if (val >= 0)
+                const int b6 = (1 << 6) - 1;
+                const int b13 = (1 << 13) - 1;
+                const int b28 = (1 << 28) - 1;
+
+                int sign = (int)((uint)value >> 31);
+
+                if ((value & ~b6) == sign * ~b6)
                 {
-                    val = val << 1;
-                    this.WriteCompressedUInt((uint)val);
+                    int n = ((value & b6) << 1) | sign;
+                    this.WriteByte((byte)n);
                 }
-                else
+                else if ((value & ~b13) == sign * ~b13)
                 {
-                    if (val > -0x40)
-                    {
-                        val = 0x40 + val;
-                        val = (val << 1) | 1;
-                        this.WriteByte((byte)val);
-                    }
-                    else if (val >= -0x2000)
-                    {
-                        val = 0x2000 - val;
-                        val = (val << 1) | 1;
-                        this.WriteByte((byte)((val >> 8) | 0x80));
-                        this.WriteByte((byte)(val & 0xff));
-                    }
-                    else if (val >= -0x20000000)
-                    {
-                        val = 0x20000000 - val;
-                        val = (val << 1) | 1;
-                        this.WriteByte((byte)((val >> 24) | 0xc0));
-                        this.WriteByte((byte)((val & 0xff0000) >> 16));
-                        this.WriteByte((byte)((val & 0xff00) >> 8));
-                        this.WriteByte((byte)(val & 0xff));
-                    }
-                    else
-                    {
-                        // ^ assume false;
-                    }
+                    int n = ((value & b13) << 1) | sign;
+                    this.WriteByte((byte)(0x80 | (n >> 8)));
+                    this.WriteByte((byte)(n & 0xff));
+                }
+                else 
+                {
+                    Debug.Assert((value & ~b28) == sign * ~b28);
+
+                    int n = ((value & b28) << 1) | sign;
+                    this.WriteByte((byte)(0xc0 | (n >> 24)));
+                    this.WriteByte((byte)((n >> 16) & 0xff));
+                    this.WriteByte((byte)((n >> 8) & 0xff));
+                    this.WriteByte((byte)(n & 0xff));
                 }
             }
         }
@@ -431,19 +424,17 @@ internal void WriteCompressedUInt(uint val)
                 }
                 else if (val <= 0x3fff)
                 {
-                    this.WriteByte((byte)((val >> 8) | 0x80));
-                    this.WriteByte((byte)(val & 0xff));
-                }
-                else if (val <= 0x1fffffff)
-                {
-                    this.WriteByte((byte)((val >> 24) | 0xc0));
-                    this.WriteByte((byte)((val & 0xff0000) >> 16));
-                    this.WriteByte((byte)((val & 0xff00) >> 8));
+                    this.WriteByte((byte)(0x80 | (val >> 8)));
                     this.WriteByte((byte)(val & 0xff));
                 }
                 else
                 {
-                    // ^ assume false;
+                    Debug.Assert(val <= 0x1fffffff);
+
+                    this.WriteByte((byte)(0xc0 | (val >> 24)));
+                    this.WriteByte((byte)((val >> 16) & 0xff));
+                    this.WriteByte((byte)((val >> 8) & 0xff));
+                    this.WriteByte((byte)(val & 0xff));
                 }
             }
         }

Src/Compilers/Core/Portable/PEWriter/PeWriter.cs

@@ -6145,7 +6145,7 @@ private void SerializeTypeReference(ITypeReference typeReference, BinaryWriter w
                     writer.WriteCompressedUInt(IteratorHelper.EnumerableCount(arrayTypeReference.LowerBounds));
                     foreach (int lowerBound in arrayTypeReference.LowerBounds)
                     {
-                        writer.WriteCompressedInt(lowerBound);
+                        writer.WriteCompressedSignedInteger(lowerBound);
                     }
 
                     return;

Src/Compilers/Core/Portable/SynthesizedLocalKind.cs

@@ -174,7 +174,7 @@ internal enum SynthesizedLocalKind
         /// <summary>
         /// All values have to be less than or equal to <see cref="MaxValidValue"/> (<see cref="EditAndContinueMethodDebugInformation"/>)
         /// </summary>
-        MaxValidValue = 0x7f - 1,
+        MaxValidValue = 0x7f - 2,
     }
 
     internal static class SynthesizedLocalKindExtensions

Src/Test/PdbUtilities/Pdb/PdbToXml.cs

@@ -538,6 +538,7 @@ private unsafe void WriteEditAndContinueLocalSlotMap(byte version, CustomDebugIn
             WriteCustomDebugInfoRecordHeaderAttributes(version, kind, size);
 
             int bodySize = size - CDI.CdiRecordHeaderSize;
+            int syntaxOffsetBaseline = -1;
 
             fixed (byte* compressedSlotMapPtr = &bytes[offset])
             {
@@ -552,6 +553,13 @@ private unsafe void WriteEditAndContinueLocalSlotMap(byte version, CustomDebugIn
                         break;
                     }
 
+                    if (b == 0xfe)
+                    {
+                        syntaxOffsetBaseline = -blobReader.ReadCompressedInteger();
+                        writer.WriteElementString("baseline", syntaxOffsetBaseline.ToString());
+                        continue;
+                    }
+
                     writer.WriteStartElement("slot");
 
                     if (b == 0)
@@ -564,10 +572,9 @@ private unsafe void WriteEditAndContinueLocalSlotMap(byte version, CustomDebugIn
                         int synthesizedKind = (b & 0x3f) - 1;
                         bool hasOrdinal = (b & (1 << 7)) != 0;
 
-                        // TODO: Right now all integers are >= -1, but we should not assume that and read Ecma335 compressed int instead.
                         int syntaxOffset;
                         bool badSyntaxOffset = !blobReader.TryReadCompressedInteger(out syntaxOffset);
-                        syntaxOffset--;
+                        syntaxOffset += syntaxOffsetBaseline;
 
                         int ordinal = 0;
                         bool badOrdinal = hasOrdinal && !blobReader.TryReadCompressedInteger(out ordinal);