Handle constants containing NUL when writing to Windows PDB (#11730)

src/Compilers/CSharp/Test/Emit/PDB/PDBConstantTests.cs

@@ -326,6 +326,109 @@ void M()
 </symbols>", format: DebugInformationFormat.PortablePdb);
         }
 
+        [Fact, WorkItem(178988, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/178988")]
+        public void TestStringWithNulCharacter_MaxSupportedLength()
+        {
+            const int length = 2031;
+            string str = new string('x', 9) + "\0" + new string('x', length - 10);
+
+            string text = @"
+class C
+{
+    void M()
+    {
+        const string x = """ + str + @""";
+    }
+}
+";
+            var c = CompileAndVerify(text, options: TestOptions.DebugDll);
+
+            c.VerifyPdb("C.M", @"
+<symbols>
+  <methods>
+    <method containingType=""C"" name=""M"">
+      <customDebugInfo>
+        <using>
+          <namespace usingCount=""0"" />
+        </using>
+      </customDebugInfo>
+      <sequencePoints>
+        <entry offset=""0x0"" startLine=""5"" startColumn=""5"" endLine=""5"" endColumn=""6"" />
+        <entry offset=""0x1"" startLine=""7"" startColumn=""5"" endLine=""7"" endColumn=""6"" />
+      </sequencePoints>
+      <scope startOffset=""0x0"" endOffset=""0x2"">
+        <constant name=""x"" value=""" + str.Replace("\0", @"\u0000") + @""" type=""String"" />
+      </scope>
+    </method>
+  </methods>
+</symbols>", format: DebugInformationFormat.Pdb);
+
+            c.VerifyPdb("C.M", @"
+<symbols>
+  <methods>
+    <method containingType=""C"" name=""M"">
+      <sequencePoints>
+        <entry offset=""0x0"" startLine=""5"" startColumn=""5"" endLine=""5"" endColumn=""6"" />
+        <entry offset=""0x1"" startLine=""7"" startColumn=""5"" endLine=""7"" endColumn=""6"" />
+      </sequencePoints>
+      <scope startOffset=""0x0"" endOffset=""0x2"">
+        <constant name=""x"" value=""" + str.Replace("\0", @"\u0000") + @""" type=""String"" />
+      </scope>
+    </method>
+  </methods>
+</symbols>", format: DebugInformationFormat.PortablePdb);
+        }
+
+        [Fact, WorkItem(178988, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/178988")]
+        public void TestStringWithNulCharacter_OverSupportedLength()
+        {
+            const int length = 2032;
+            string str = new string('x', 9) + "\0" + new string('x', length - 10);
+
+            string text = @"
+class C
+{
+    void M()
+    {
+        const string x = """ + str + @""";
+    }
+}
+";
+            var c = CompileAndVerify(text, options: TestOptions.DebugDll);
+
+            c.VerifyPdb("C.M", @"
+<symbols>
+  <methods>
+    <method containingType=""C"" name=""M"">
+      <customDebugInfo>
+        <using>
+          <namespace usingCount=""0"" />
+        </using>
+      </customDebugInfo>
+      <sequencePoints>
+        <entry offset=""0x0"" startLine=""5"" startColumn=""5"" endLine=""5"" endColumn=""6"" />
+        <entry offset=""0x1"" startLine=""7"" startColumn=""5"" endLine=""7"" endColumn=""6"" />
+      </sequencePoints>
+    </method>
+  </methods>
+</symbols>", format: DebugInformationFormat.Pdb);
+
+            c.VerifyPdb("C.M", @"
+<symbols>
+  <methods>
+    <method containingType=""C"" name=""M"">
+      <sequencePoints>
+        <entry offset=""0x0"" startLine=""5"" startColumn=""5"" endLine=""5"" endColumn=""6"" />
+        <entry offset=""0x1"" startLine=""7"" startColumn=""5"" endLine=""7"" endColumn=""6"" />
+      </sequencePoints>
+      <scope startOffset=""0x0"" endOffset=""0x2"">
+        <constant name=""x"" value=""" + str.Replace("\0", @"\u0000") + @""" type=""String"" />
+      </scope>
+    </method>
+  </methods>
+</symbols>", format: DebugInformationFormat.PortablePdb);
+        }
+
         [Fact]
         public void TestDecimalLocalConstants()
         {

src/Compilers/Core/Portable/NativePdbWriter/PdbWriter.cs

@@ -1270,17 +1270,32 @@ private void DefineLocalStringConstant(string name, string value, uint constantS
         {
             Debug.Assert(value != null);
 
+            int encodedLength;
+
             // ISymUnmanagedWriter2 doesn't handle unicode strings with unmatched unicode surrogates.
             // We use the .NET UTF8 encoder to replace unmatched unicode surrogates with unicode replacement character.
+
             if (!MetadataHelpers.IsValidUnicodeString(value))
             {
                 byte[] bytes = Encoding.UTF8.GetBytes(value);
+                encodedLength = bytes.Length;
                 value = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
             }
+            else
+            {
+                encodedLength = Encoding.UTF8.GetByteCount(value);
+            }
 
-            // EDMAURER If defining a string constant and it is too long (length limit is undocumented), this method throws
-            // an ArgumentException.
-            // (see EMITTER::EmitDebugLocalConst)
+            // +1 for terminating NUL character
+            encodedLength++;
+
+            // If defining a string constant and it is too long (length limit is not documented by the API), DefineConstant2 throws an ArgumentException.
+            // However, diasymreader doesn't calculate the length correctly in presence of NUL characters in the string.
+            // Until that's fixed we need to check the limit ourselves. See http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/178988
+            if (encodedLength > 2032)
+            {
+                return;
+            }
 
             try
             {