Merge pull request #1206 from cston/1105626

EE: Non-printable characters should not be escaped when using 'nq' format specifier

src/Compilers/CSharp/Portable/SymbolDisplay/ObjectDisplay.cs

@@ -133,115 +133,115 @@ internal static string FormatLiteral(bool value)
             return value ? "true" : "false";
         }
 
-        internal static string FormatString(string str, char quote, bool escapeNonPrintable)
+        private static void FormatStringChar(
+            ref PooledStringBuilder pooledBuilder,
+            string str,
+            int index,
+            char c,
+            char quote,
+            bool useLanguageSpecificEscapes,
+            bool useUnicodeEscapes)
         {
-            PooledStringBuilder pooledBuilder = null;
-            StringBuilder sb = null;
-            int lastEscape = -1;
-            for (int i = 0; i < str.Length; i++)
-            {
-                char c = str[i];
-                char replaceWith = '\0';
-                bool unicodeEscape = false;
-                switch (c)
-                {
-                    case '\\':
-                        replaceWith = c;
-                        break;
-
-                    case '\0':
-                        replaceWith = '0';
-                        break;
-
-                    case '\r':
-                        replaceWith = 'r';
-                        break;
-
-                    case '\n':
-                        replaceWith = 'n';
-                        break;
-
-                    case '\t':
-                        replaceWith = 't';
-                        break;
-
-                    case '\b':
-                        replaceWith = 'b';
-                        break;
-
-                    case '\v':
-                        replaceWith = 'v';
-                        break;
-
-                    default:
-                        if (quote == c)
-                        {
-                            replaceWith = c;
-                            break;
-                        }
-
-                        if (escapeNonPrintable)
-                        {
-                            switch (CharUnicodeInfo.GetUnicodeCategory(c))
-                            {
-                                case UnicodeCategory.OtherNotAssigned:
-                                case UnicodeCategory.ParagraphSeparator:
-                                case UnicodeCategory.Control:
-                                    unicodeEscape = true;
-                                    break;
-                            }
-                        }
+            Debug.Assert(quote == 0 || quote == '"' || quote == '\'');
 
-                        break;
-                }
-
-                if (unicodeEscape || replaceWith != '\0')
-                {
-                    if (pooledBuilder == null)
+            string replaceWith = null;
+            bool unicodeEscape = false;
+            switch (CharUnicodeInfo.GetUnicodeCategory(c))
+            {
+                case UnicodeCategory.OtherPunctuation:
+                    if (useLanguageSpecificEscapes)
                     {
-                        pooledBuilder = PooledStringBuilder.GetInstance();
-                        sb = pooledBuilder.Builder;
-                        if (quote != 0)
+                        switch (c)
                         {
-                            sb.Append(quote);
+                            case '\\':
+                                replaceWith = "\\\\";
+                                break;
+                            case '"':
+                                if (quote == c)
+                                {
+                                    replaceWith = "\\\"";
+                                }
+                                break;
+                            case '\'':
+                                if (quote == c)
+                                {
+                                    replaceWith = "\\'";
+                                }
+                                break;
                         }
                     }
-
-                    sb.Append(str, lastEscape + 1, i - (lastEscape + 1));
-                    sb.Append('\\');
-                    if (unicodeEscape)
+                    break;
+                case UnicodeCategory.OtherNotAssigned:
+                case UnicodeCategory.ParagraphSeparator:
+                    if (useUnicodeEscapes)
                     {
-                        sb.Append('u');
-                        sb.Append(((int)c).ToString("x4"));
+                        unicodeEscape = true;
                     }
-                    else
+                    break;
+                case UnicodeCategory.Control:
+                    if (useLanguageSpecificEscapes)
                     {
-                        sb.Append(replaceWith);
+                        switch (c)
+                        {
+                            case '\0':
+                                replaceWith = "\\0";
+                                break;
+                            case '\a':
+                                replaceWith = "\\a";
+                                break;
+                            case '\b':
+                                replaceWith = "\\b";
+                                break;
+                            case '\f':
+                                replaceWith = "\\f";
+                                break;
+                            case '\n':
+                                replaceWith = "\\n";
+                                break;
+                            case '\r':
+                                replaceWith = "\\r";
+                                break;
+                            case '\t':
+                                replaceWith = "\\t";
+                                break;
+                            case '\v':
+                                replaceWith = "\\v";
+                                break;
+                        }
                     }
-
-                    lastEscape = i;
-                }
+                    if ((replaceWith == null) && useUnicodeEscapes)
+                    {
+                        unicodeEscape = true;
+                    }
+                    break;
             }
 
-            if (sb != null)
+            if ((replaceWith != null) || unicodeEscape)
             {
-                sb.Append(str, lastEscape + 1, str.Length - (lastEscape + 1));
-                if (quote != 0)
+                if (pooledBuilder == null)
                 {
-                    sb.Append(quote);
+                    pooledBuilder = PooledStringBuilder.GetInstance();
+                    if (index > 0)
+                    {
+                        pooledBuilder.Builder.Append(str, 0, index);
+                    }
+                }
+                var builder = pooledBuilder.Builder;
+                if (replaceWith != null)
+                {
+                    builder.Append(replaceWith);
+                }
+                else
+                {
+                    builder.Append("\\u");
+                    builder.Append(((int)c).ToString("x4"));
                 }
-
-                return pooledBuilder.ToStringAndFree();
             }
-
-            switch (quote)
+            else if (pooledBuilder != null)
             {
-                case '"': return String.Concat("\"", str, "\"");
-                case '\'': return String.Concat("'", str, "'");
-                case '\0': return str;
+                var builder = pooledBuilder.Builder;
+                builder.Append(c);
             }
-
-            throw ExceptionUtilities.Unreachable;
         }
 
         /// <summary>
@@ -262,7 +262,43 @@ public static string FormatLiteral(string value, ObjectDisplayOptions options)
                 throw new ArgumentNullException("value");
             }
 
-            return FormatString(value, options.IncludesOption(ObjectDisplayOptions.UseQuotes) ? '"' : '\0', escapeNonPrintable: true);
+            var quote = options.IncludesOption(ObjectDisplayOptions.UseQuotes) ? '"' : '\0';
+            PooledStringBuilder pooledBuilder = null;
+            StringBuilder builder = null;
+            if (quote != 0)
+            {
+                pooledBuilder = PooledStringBuilder.GetInstance();
+                builder = pooledBuilder.Builder;
+                builder.Append(quote);
+            }
+            for (int i = 0; i < value.Length; i++)
+            {
+                FormatStringChar(ref pooledBuilder, value, i, value[i], quote, useLanguageSpecificEscapes: true, useUnicodeEscapes: true);
+            }
+            if (quote != 0)
+            {
+                builder.Append(quote);
+            }
+            return (pooledBuilder == null) ? value : pooledBuilder.ToStringAndFree();
+        }
+
+        internal static string FormatString(string str, bool useQuotes)
+        {
+            if (!useQuotes)
+            {
+                return str;
+            }
+
+            var pooledBuilder = PooledStringBuilder.GetInstance();
+            var builder = pooledBuilder.Builder;
+            const char quote = '"';
+            builder.Append(quote);
+            for (int i = 0; i < str.Length; i++)
+            {
+                FormatStringChar(ref pooledBuilder, str, i, str[i], quote, useLanguageSpecificEscapes: useQuotes, useUnicodeEscapes: false);
+            }
+            builder.Append(quote);
+            return pooledBuilder.ToStringAndFree();
         }
 
         /// <summary>
@@ -273,17 +309,28 @@ public static string FormatLiteral(string value, ObjectDisplayOptions options)
         /// <returns>A character literal with the given value.</returns>
         internal static string FormatLiteral(char c, ObjectDisplayOptions options)
         {
+            var useQuotes = options.IncludesOption(ObjectDisplayOptions.UseQuotes);
+            var quote = useQuotes ? '\'' : '\0';
             var includeCodePoints = options.IncludesOption(ObjectDisplayOptions.IncludeCodePoints);
-            var result = FormatString(c.ToString(),
-                                      quote: options.IncludesOption(ObjectDisplayOptions.UseQuotes) ? '\'' : '\0',
-                                      escapeNonPrintable: !includeCodePoints);
+
+            var pooledBuilder = PooledStringBuilder.GetInstance();
+            var builder = pooledBuilder.Builder;
             if (includeCodePoints)
             {
-                var codepoint = options.IncludesOption(ObjectDisplayOptions.UseHexadecimalNumbers) ? "0x" + ((int)c).ToString("x4") : ((int)c).ToString();
-                return codepoint + " " + result;
+                builder.Append(options.IncludesOption(ObjectDisplayOptions.UseHexadecimalNumbers) ? "0x" + ((int)c).ToString("x4") : ((int)c).ToString());
+                builder.Append(" ");
             }
-
-            return result;
+            if (useQuotes)
+            {
+                builder.Append(quote);
+            }
+            var escapeNonPrintable = !includeCodePoints;
+            FormatStringChar(ref pooledBuilder, null, 0, c, quote, useLanguageSpecificEscapes: useQuotes, useUnicodeEscapes: !includeCodePoints);
+            if (useQuotes)
+            {
+                builder.Append(quote);
+            }
+            return pooledBuilder.ToStringAndFree();
         }
 
         internal static string FormatLiteral(sbyte value, ObjectDisplayOptions options)

src/Compilers/CSharp/Test/Symbol/SymbolDisplay/ObjectDisplayTests.cs

@@ -121,34 +121,29 @@ public void Characters()
             Assert.Equal("0x000a '\\n'", ObjectDisplay.FormatLiteral('\n', ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.IncludeCodePoints | ObjectDisplayOptions.UseHexadecimalNumbers));
             Assert.Equal("0x000b '\\v'", ObjectDisplay.FormatLiteral('\v', ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.IncludeCodePoints | ObjectDisplayOptions.UseHexadecimalNumbers));
             Assert.Equal("0x000d '\\r'", ObjectDisplay.FormatLiteral('\r', ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.IncludeCodePoints | ObjectDisplayOptions.UseHexadecimalNumbers));
+            Assert.Equal("0x000d \r", ObjectDisplay.FormatLiteral('\r', ObjectDisplayOptions.IncludeCodePoints | ObjectDisplayOptions.UseHexadecimalNumbers));
         }
 
         [Fact]
         public void Strings()
         {
-            Assert.Equal("", ObjectDisplay.FormatString("", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"a", ObjectDisplay.FormatString(@"a", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"ab", ObjectDisplay.FormatString(@"ab", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"\\", ObjectDisplay.FormatString(@"\", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"\\a", ObjectDisplay.FormatString(@"\a", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"a\\b", ObjectDisplay.FormatString(@"a\b", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"ab\\c", ObjectDisplay.FormatString(@"ab\c", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"ab\\cd", ObjectDisplay.FormatString(@"ab\cd", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"ab\\cd\\", ObjectDisplay.FormatString(@"ab\cd\", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"ab\\cd\\e", ObjectDisplay.FormatString(@"ab\cd\e", quote: '\0', escapeNonPrintable: true));
-            Assert.Equal(@"\\\\\\\\", ObjectDisplay.FormatString(@"\\\\", quote: '\0', escapeNonPrintable: true));
-
-            Assert.Equal(@"""""", ObjectDisplay.FormatString("", quote: '"', escapeNonPrintable: true));
-            Assert.Equal(@"""\""\""""", ObjectDisplay.FormatString(@"""""", quote: '"', escapeNonPrintable: true));
-            Assert.Equal(@"""'""", ObjectDisplay.FormatString(@"'", quote: '"', escapeNonPrintable: true));
-            Assert.Equal(@"""ab""", ObjectDisplay.FormatString(@"ab", quote: '"', escapeNonPrintable: true));
-            Assert.Equal(@"""\\""", ObjectDisplay.FormatString(@"\", quote: '"', escapeNonPrintable: true));
-
-            Assert.Equal(@"''", ObjectDisplay.FormatString("", quote: '\'', escapeNonPrintable: true));
-            Assert.Equal(@"'\'\''", ObjectDisplay.FormatString("''", quote: '\'', escapeNonPrintable: true));
-            Assert.Equal(@"'""'", ObjectDisplay.FormatString(@"""", quote: '\'', escapeNonPrintable: true));
-            Assert.Equal(@"'ab'", ObjectDisplay.FormatString(@"ab", quote: '\'', escapeNonPrintable: true));
-            Assert.Equal(@"'\\'", ObjectDisplay.FormatString(@"\", quote: '\'', escapeNonPrintable: true));
+            Assert.Equal("", ObjectDisplay.FormatString("", useQuotes: false));
+            Assert.Equal(@"a", ObjectDisplay.FormatString(@"a", useQuotes: false));
+            Assert.Equal(@"ab", ObjectDisplay.FormatString(@"ab", useQuotes: false));
+            Assert.Equal(@"\", ObjectDisplay.FormatString(@"\", useQuotes: false));
+            Assert.Equal(@"\a", ObjectDisplay.FormatString(@"\a", useQuotes: false));
+            Assert.Equal(@"a\b", ObjectDisplay.FormatString(@"a\b", useQuotes: false));
+            Assert.Equal(@"ab\c", ObjectDisplay.FormatString(@"ab\c", useQuotes: false));
+            Assert.Equal(@"ab\cd", ObjectDisplay.FormatString(@"ab\cd", useQuotes: false));
+            Assert.Equal(@"ab\cd\", ObjectDisplay.FormatString(@"ab\cd\", useQuotes: false));
+            Assert.Equal(@"ab\cd\e", ObjectDisplay.FormatString(@"ab\cd\e", useQuotes: false));
+            Assert.Equal(@"\\\\", ObjectDisplay.FormatString(@"\\\\", useQuotes: false));
+
+            Assert.Equal(@"""""", ObjectDisplay.FormatString("", useQuotes: true));
+            Assert.Equal(@"""\""\""""", ObjectDisplay.FormatString(@"""""", useQuotes: true));
+            Assert.Equal(@"""'""", ObjectDisplay.FormatString(@"'", useQuotes: true));
+            Assert.Equal(@"""ab""", ObjectDisplay.FormatString(@"ab", useQuotes: true));
+            Assert.Equal(@"""\\""", ObjectDisplay.FormatString(@"\", useQuotes: true));
 
             Assert.Equal("\"x\"", ObjectDisplay.FormatLiteral("x", ObjectDisplayOptions.UseQuotes));
             Assert.Equal("x", ObjectDisplay.FormatLiteral("x", ObjectDisplayOptions.None));
@@ -158,24 +153,42 @@ public void Strings()
             {
                 sb.Append((char)i);
             }
-
-            var s = ObjectDisplay.FormatString(sb.ToString(), quote: '"', escapeNonPrintable: true);
+            var s = sb.ToString();
+
+            // Formatting with quotes should escape specific control characters.
+            var expected =
+                "\"\\0\u0001\u0002\u0003\u0004\u0005\u0006\\a\\b\\t\\n\\v\\f\\r\u000e\u000f\u0010" +
+                "\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d" +
+                "\u001e\u001f !\\\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[" +
+                "\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f\u0080\u0081\u0082\u0083\u0084\u0085" +
+                "\u0086\u0087\u0088\u0089\u008a\u008b\u008c\u008d\u008e\u008f\u0090\u0091\u0092" +
+                "\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009a\u009b\u009c\u009d\u009e\u009f" +
+                " ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèé" +
+                "êëìíîïðñòóôõö÷øùúûüýþ\"";
+            Assert.Equal(
+                expected,
+                ObjectDisplay.FormatString(s, useQuotes: true));
+
+            // Formatting without quotes should not escape any characters.
+            expected =
+                "\0\u0001\u0002\u0003\u0004\u0005\u0006\a\u0008\u0009\u000a\u000b\f\u000d\u000e\u000f\u0010" +
+                "\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d" +
+                "\u001e\u001f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[" +
+                "\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f\u0080\u0081\u0082\u0083\u0084\u0085" +
+                "\u0086\u0087\u0088\u0089\u008a\u008b\u008c\u008d\u008e\u008f\u0090\u0091\u0092" +
+                "\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009a\u009b\u009c\u009d\u009e\u009f" +
+                " ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèé" +
+                "êëìíîïðñòóôõö÷øùúûüýþ";
             Assert.Equal(
-                @"""\0\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\v\u000c\r\u000e\u000f\u0010" +
-                @"\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d" +
-                @"\u001e\u001f !\""#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[" +
-                @"\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f\u0080\u0081\u0082\u0083\u0084\u0085" +
-                @"\u0086\u0087\u0088\u0089\u008a\u008b\u008c\u008d\u008e\u008f\u0090\u0091\u0092" +
-                @"\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009a\u009b\u009c\u009d\u009e\u009f" +
-                @" ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèé" +
-                @"êëìíîïðñòóôõö÷øùúûüýþ""", s);
+                expected,
+                ObjectDisplay.FormatString(s, useQuotes: false));
 
             var arabic = "انتخابات مبكرة، بعد يوم حافل بالاحداث السياسية، بعد";
-            s = ObjectDisplay.FormatString(arabic, quote: '\0', escapeNonPrintable: true);
+            s = ObjectDisplay.FormatString(arabic, useQuotes: false);
             Assert.Equal(arabic, s);
 
             var hebrew = "והמנהלים רפואיים של ארבעת קופות החולים. בסיום הפגישה הבהיר";
-            s = ObjectDisplay.FormatString(hebrew, quote: '\0', escapeNonPrintable: true);
+            s = ObjectDisplay.FormatString(hebrew, useQuotes: false);
             Assert.Equal(hebrew, s);
         }
 

src/Compilers/CSharp/Test/Syntax/Syntax/SyntaxFactoryTests.cs

@@ -227,13 +227,16 @@ public void TestLiteralDefaultStringValues()
             // string
             CheckLiteralToString("A", @"""A""");
             CheckLiteralToString("\r", @"""\r""");
-            CheckLiteralToString("\u0007", @"""\u0007""");
+            CheckLiteralToString("\u0007", @"""\a""");
+            CheckLiteralToString("\u000c", @"""\f""");
+            CheckLiteralToString("\u001f", @"""\u001f""");
 
             // char
             CheckLiteralToString('A', @"'A'");
             CheckLiteralToString('\r', @"'\r'");
-            CheckLiteralToString('\u0007', @"'\u0007'");
-
+            CheckLiteralToString('\u0007', @"'\a'");
+            CheckLiteralToString('\u000c', @"'\f'");
+            CheckLiteralToString('\u001f', @"'\u001f'");
 
             // byte
             CheckLiteralToString(byte.MinValue, @"0");

src/Compilers/VisualBasic/Portable/SymbolDisplay/ObjectDisplay.vb

@@ -122,7 +122,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ObjectDisplay
         ''' </summary>
         ''' <param name="value">Literal value.</param>
         ''' <param name="options">Options used to customize formatting of a literal value.</param>
-        Friend Function FormatLiteral(value As String, options As ObjectDisplayOptions, Optional nonPrintableSubstitute As Char = Nothing) As String
+        Friend Function FormatLiteral(value As String, options As ObjectDisplayOptions) As String
             ValidateOptions(options)
 
             If value Is Nothing Then
@@ -131,9 +131,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ObjectDisplay
 
             Dim pooledBuilder = PooledStringBuilder.GetInstance()
             Dim sb = pooledBuilder.Builder
+            Dim useQuotes = options.IncludesOption(ObjectDisplayOptions.UseQuotes)
+            Dim useHex = options.IncludesOption(ObjectDisplayOptions.UseHexadecimalNumbers)
 
-            For Each token As Integer In TokenizeString(value, options.IncludesOption(ObjectDisplayOptions.UseQuotes), nonPrintableSubstitute,
-                                                        options.IncludesOption(ObjectDisplayOptions.UseHexadecimalNumbers))
+            For Each token As Integer In TokenizeString(value, useQuotes, useHex)
                 sb.Append(ChrW(token And &HFFFF)) ' lower 16 bits of token contains the Unicode char value
             Next
 
@@ -143,6 +144,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ObjectDisplay
         Friend Function FormatLiteral(c As Char, options As ObjectDisplayOptions) As String
             ValidateOptions(options)
 
+            If Not options.IncludesOption(ObjectDisplayOptions.UseQuotes) Then
+                Return c.ToString()
+            End If
+
             Dim wellKnown = GetWellKnownCharacterName(c)
             If wellKnown IsNot Nothing Then
                 Return wellKnown
@@ -153,11 +158,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ObjectDisplay
                 Return If(options.IncludesOption(ObjectDisplayOptions.UseHexadecimalNumbers), "ChrW(&H" & codepoint.ToString("X"), "ChrW(" & codepoint.ToString()) & ")"
             End If
 
-            If options.IncludesOption(ObjectDisplayOptions.UseQuotes) Then
-                Return """"c & EscapeQuote(c) & """"c & "c"
-            Else
-                Return c
-            End If
+            Return """"c & EscapeQuote(c) & """"c & "c"
         End Function
 
         Private Function EscapeQuote(c As Char) As String
@@ -361,7 +362,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ObjectDisplay
         End Function
 
         ' TODO: consider making "token" returned by this function a structure to abstract bit masking operations
-        Friend Iterator Function TokenizeString(str As String, quote As Boolean, nonPrintableSubstitute As Char, useHexadecimalNumbers As Boolean) As IEnumerable(Of Integer)
+        Friend Iterator Function TokenizeString(str As String, quote As Boolean, useHexadecimalNumbers As Boolean) As IEnumerable(Of Integer)
             If str.Length = 0 Then
                 If quote Then
                     Yield Quotes()
@@ -395,13 +396,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ObjectDisplay
                 End If
 
                 If isNonPrintable Then
-                    If nonPrintableSubstitute <> s_nullChar Then
-                        Yield Character(nonPrintableSubstitute)
-
-                        If isCrLf Then
-                            Yield Character(nonPrintableSubstitute)
-                        End If
-                    ElseIf quote Then
+                    If quote Then
                         If lastConcatenandWasQuoted Then
                             Yield Quotes()
                             lastConcatenandWasQuoted = False

src/Compilers/VisualBasic/Portable/SymbolDisplay/SymbolDisplay.vb

@@ -135,7 +135,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
             Dim sb = pooledBuilder.Builder
 
             Dim lastKind = -1
-            For Each token As Integer In ObjectDisplay.TokenizeString(str, quote:=True, nonPrintableSubstitute:=Nothing, useHexadecimalNumbers:=True)
+            For Each token As Integer In ObjectDisplay.TokenizeString(str, quote:=True, useHexadecimalNumbers:=True)
                 Dim kind = token >> 16
 
                 ' merge contiguous tokens of the same kind into a single part

src/Compilers/VisualBasic/Test/Symbol/SymbolDisplay/ObjectDisplayTests.vb

@@ -101,9 +101,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.UnitTests
 
         <Fact>
         Public Sub Characters()
-            ' Note - this is significanlty different from what Dev10 VB EE does, which is
-            ' - ignore "nq" setting
-            ' - print non-printable characters, no escaping
+            ' Note, the legacy EE ignores the "nq" setting
             Assert.Equal("""x""c", FormatPrimitive("x"c, quoteStrings:=True))
             Assert.Equal("x", FormatPrimitive("x"c, quoteStrings:=False))
             Assert.Equal("""x""c", FormatPrimitiveUsingHexadecimalNumbers("x"c, quoteStrings:=True))
@@ -111,10 +109,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.UnitTests
 
             Assert.Equal("vbNullChar", FormatPrimitiveUsingHexadecimalNumbers(ChrW(0), quoteStrings:=True))
             Assert.Equal("ChrW(&H1E)", FormatPrimitiveUsingHexadecimalNumbers(ChrW(&H1E), quoteStrings:=True))
-            Assert.Equal("ChrW(&H1E)", FormatPrimitiveUsingHexadecimalNumbers(ChrW(&H1E), quoteStrings:=False))
-            Assert.Equal("ChrW(20)", FormatPrimitive(ChrW(20)))
+            Assert.Equal(New String({ChrW(&H1E)}), FormatPrimitiveUsingHexadecimalNumbers(ChrW(&H1E), quoteStrings:=False))
+            Assert.Equal(New String({ChrW(20)}), FormatPrimitive(ChrW(20)))
             Assert.Equal("vbBack", FormatPrimitiveUsingHexadecimalNumbers(ChrW(&H8), quoteStrings:=True))
-            Assert.Equal("vbBack", FormatPrimitive(ChrW(&H8)))
+            Assert.Equal(vbBack, FormatPrimitive(ChrW(&H8)))
             Assert.Equal("vbLf", FormatPrimitiveUsingHexadecimalNumbers(ChrW(&HA), quoteStrings:=True))
             Assert.Equal("vbVerticalTab", FormatPrimitiveUsingHexadecimalNumbers(vbVerticalTab(0), quoteStrings:=True))
             Assert.Equal("vbTab", FormatPrimitiveUsingHexadecimalNumbers(ChrW(&H9), quoteStrings:=True))
@@ -132,8 +130,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.UnitTests
 
             Assert.Equal("ChrW(&HFFFE)", FormatPrimitiveUsingHexadecimalNumbers(ChrW(&HFFFE), quoteStrings:=True))
             Assert.Equal("ChrW(65534)", FormatPrimitive(ChrW(&HFFFE), quoteStrings:=True))
-            Assert.Equal("ChrW(&HFFFE)", FormatPrimitiveUsingHexadecimalNumbers(ChrW(&HFFFE), quoteStrings:=False))
-            Assert.Equal("ChrW(65534)", FormatPrimitive(ChrW(&HFFFE), quoteStrings:=False))
+            Assert.Equal(New String({ChrW(&HFFFE)}), FormatPrimitiveUsingHexadecimalNumbers(ChrW(&HFFFE), quoteStrings:=False))
+            Assert.Equal(New String({ChrW(65534)}), FormatPrimitive(ChrW(&HFFFE), quoteStrings:=False))
 
             Dim s = "a" & ChrW(&HFFFF) & ChrW(&HFFFE) & vbCrLf & "b"
 
@@ -144,10 +142,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.UnitTests
             Assert.Equal("ChrW(&HFFFF) & ""a"" & ChrW(&HFFFE)", FormatPrimitiveUsingHexadecimalNumbers(ChrW(&HFFFF) & "a" & ChrW(&HFFFE), quoteStrings:=True))
             Assert.Equal("""a"" & ChrW(&HFFFF) & ChrW(&HFFFE) & vbCrLf & ""b""", FormatPrimitiveUsingHexadecimalNumbers(s, quoteStrings:=True))
 
-            ' non-printable characters are replaced by spaces if quoting is disabled
+            ' non-printable characters are unchanged if quoting is disabled
             Assert.Equal(s, FormatPrimitiveUsingHexadecimalNumbers(s, quoteStrings:=False))
-            Assert.Equal("a....b", ObjectDisplay.FormatLiteral(s, ObjectDisplayOptions.None, nonPrintableSubstitute:="."c))
-            Assert.Equal("""a....b""", ObjectDisplay.FormatLiteral(s, ObjectDisplayOptions.UseQuotes Or ObjectDisplayOptions.UseHexadecimalNumbers, nonPrintableSubstitute:="."c))
+            Assert.Equal(s, ObjectDisplay.FormatLiteral(s, ObjectDisplayOptions.None))
+            Assert.Equal("""a"" & ChrW(&HFFFF) & ChrW(&HFFFE) & vbCrLf & ""b""", ObjectDisplay.FormatLiteral(s, ObjectDisplayOptions.UseQuotes Or ObjectDisplayOptions.UseHexadecimalNumbers))
 
             ' "well-known" characters:
             Assert.Equal("""a"" & vbBack", FormatPrimitiveUsingHexadecimalNumbers("a" & vbBack, quoteStrings:=True))

src/ExpressionEvaluator/CSharp/Source/ResultProvider/CSharpFormatter.Values.cs

@@ -211,7 +211,7 @@ internal override string FormatPrimitiveObject(object value, ObjectDisplayOption
 
         internal override string FormatString(string str, ObjectDisplayOptions options)
         {
-            return ObjectDisplay.FormatString(str, quote: options.IncludesOption(ObjectDisplayOptions.UseQuotes) ? '"' : '\0', escapeNonPrintable: false);
+            return ObjectDisplay.FormatString(str, useQuotes: options.IncludesOption(ObjectDisplayOptions.UseQuotes));
         }
     }
 }

src/ExpressionEvaluator/CSharp/Test/ResultProvider/ExpansionTests.cs

@@ -1676,10 +1676,16 @@ public void UnicodeChar()
                 EvalResult("c", "4660 '\u1234'", "char", "c", editableValue: "'\u1234'"));
 
             // This char is not printable, so we expect the EditableValue to be the unicode escape representation.
-            value = CreateDkmClrValue('\u0007', typeof(char));
+            value = CreateDkmClrValue('\u001f');
             evalResult = FormatResult("c", value, inspectionContext: CreateDkmInspectionContext(radix: 16));
             Verify(evalResult,
-                EvalResult("c", "0x0007 '\u0007'", "char", "c", editableValue: "'\\u0007'"));
+                EvalResult("c", "0x001f '\u001f'", "char", "c", editableValue: "'\\u001f'"));
+
+            // This char is not printable, but there is a specific escape character.
+            value = CreateDkmClrValue('\u0007');
+            evalResult = FormatResult("c", value, inspectionContext: CreateDkmInspectionContext(radix: 16));
+            Verify(evalResult,
+                EvalResult("c", "0x0007 '\\a'", "char", "c", editableValue: "'\\a'"));
         }
 
         [WorkItem(1002381)]

src/ExpressionEvaluator/CSharp/Test/ResultProvider/FormatSpecifierTests.cs

@@ -40,6 +40,24 @@ public void NoQuotes_String()
             Verify(evalResult,
                 EvalResult("s", "\"", "string", "s", editableValue: "\"\\\"\"", flags: DkmEvaluationResultFlags.RawString));
 
+            // "\\"
+            value = CreateDkmClrValue("\\", type: stringType);
+            evalResult = FormatResult("s", value, inspectionContext: inspectionContext);
+            Verify(evalResult,
+                EvalResult("s", "\\", "string", "s", editableValue: "\"\\\\\"", flags: DkmEvaluationResultFlags.RawString));
+
+            // "a\r\n\t\v\b\u001eb"
+            value = CreateDkmClrValue("a\r\n\tb\v\b\u001ec", type: stringType);
+            evalResult = FormatResult("s", value, inspectionContext: inspectionContext);
+            Verify(evalResult,
+                EvalResult("s", "a\r\n\tb\v\b\u001ec", "string", "s", editableValue: "\"a\\r\\n\\tb\\v\\b\u001ec\"", flags: DkmEvaluationResultFlags.RawString));
+
+            // "a\0b"
+            value = CreateDkmClrValue("a\0b", type: stringType);
+            evalResult = FormatResult("s", value, inspectionContext: inspectionContext);
+            Verify(evalResult,
+                EvalResult("s", "a\0b", "string", "s", editableValue: "\"a\\0b\"", flags: DkmEvaluationResultFlags.RawString));
+
             // " " with alias
             value = CreateDkmClrValue(" ", type: stringType, alias: "1", evalFlags: DkmEvaluationResultFlags.HasObjectId);
             evalResult = FormatResult("s", value, inspectionContext: inspectionContext);
@@ -68,7 +86,7 @@ public void NoQuotes_Char()
             var value = CreateDkmClrValue((char)0, type: charType);
             var evalResult = FormatResult("c", value, inspectionContext: inspectionContext);
             Verify(evalResult,
-                EvalResult("c", "0 \\0", "char", "c", editableValue: "'\\0'", flags: DkmEvaluationResultFlags.None));
+                EvalResult("c", "0 \0", "char", "c", editableValue: "'\\0'", flags: DkmEvaluationResultFlags.None));
 
             // '\''
             value = CreateDkmClrValue('\'', type: charType);
@@ -82,6 +100,24 @@ public void NoQuotes_Char()
             Verify(evalResult,
                 EvalResult("c", "34 \"", "char", "c", editableValue: "'\"'", flags: DkmEvaluationResultFlags.None));
 
+            // '\\'
+            value = CreateDkmClrValue('\\', type: charType);
+            evalResult = FormatResult("c", value, inspectionContext: inspectionContext);
+            Verify(evalResult,
+                EvalResult("c", "92 \\", "char", "c", editableValue: "'\\\\'", flags: DkmEvaluationResultFlags.None));
+
+            // '\n'
+            value = CreateDkmClrValue('\n', type: charType);
+            evalResult = FormatResult("c", value, inspectionContext: inspectionContext);
+            Verify(evalResult,
+                EvalResult("c", "10 \n", "char", "c", editableValue: "'\\n'", flags: DkmEvaluationResultFlags.None));
+
+            // '\u001e'
+            value = CreateDkmClrValue('\u001e', type: charType);
+            evalResult = FormatResult("c", value, inspectionContext: inspectionContext);
+            Verify(evalResult,
+                EvalResult("c", "30 \u001e", "char", "c", editableValue: "'\\u001e'", flags: DkmEvaluationResultFlags.None));
+
             // array
             value = CreateDkmClrValue(new char[] { '1' }, type: charType.MakeArrayType());
             evalResult = FormatResult("a", value, inspectionContext: inspectionContext);

src/ExpressionEvaluator/CSharp/Test/ResultProvider/ValueFormattingTests.cs

@@ -94,12 +94,18 @@ public void Char()
                     case '\t':
                         expected = "\\t";
                         break;
+                    case '\f':
+                        expected = "\\f";
+                        break;
                     case '\r':
                         expected = "\\r";
                         break;
                     case '\n':
                         expected = "\\n";
                         break;
+                    case '\a':
+                        expected = "\\a";
+                        break;
                     case '\b':
                         expected = "\\b";
                         break;
@@ -150,12 +156,18 @@ public void String()
                     case '\t':
                         expected = "\\t";
                         break;
+                    case '\f':
+                        expected = "\\f";
+                        break;
                     case '\r':
                         expected = "\\r";
                         break;
                     case '\n':
                         expected = "\\n";
                         break;
+                    case '\a':
+                        expected = "\\a";
+                        break;
                     case '\b':
                         expected = "\\b";
                         break;

src/ExpressionEvaluator/VisualBasic/Test/ResultProvider/ExpansionTests.vb

@@ -280,7 +280,7 @@ End Class
 
             ' This Char is not printable, so we expect the EditableValue to be the "ChrW" representation.
             quotedChar = "ChrW(&H7)"
-            value = CreateDkmClrValue(ChrW(&H0007), GetType(Char))
+            value = CreateDkmClrValue(ChrW(&H0007))
             result = FormatResult("c", value, inspectionContext:=CreateDkmInspectionContext(radix:=16))
             Verify(result,
                 EvalResult("c", quotedChar, "Char", "c", editableValue:=quotedChar))

src/ExpressionEvaluator/VisualBasic/Test/ResultProvider/FormatSpecifierTests.vb

@@ -40,6 +40,18 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ExpressionEvaluator
             Verify(result,
                    EvalResult("s", """", "String", "s", editableValue:="""""""""", flags:=DkmEvaluationResultFlags.RawString))
 
+            ' "a" & vbCrLf & "b" & vbTab & vbVerticalTab & vbBack & "c"
+            value = CreateDkmClrValue("a" & vbCrLf & "b" & vbTab & vbVerticalTab & vbBack & "c", type:=stringType)
+            result = FormatResult("s", value, inspectionContext:=inspectionContext)
+            Verify(result,
+                   EvalResult("s", "a" & vbCrLf & "b" & vbTab & vbVerticalTab & vbBack & "c", "String", "s", editableValue:="""a"" & vbCrLf & ""b"" & vbTab & vbVerticalTab & vbBack & ""c""", flags:=DkmEvaluationResultFlags.RawString))
+
+            ' "a" & vbNullChar & "b"
+            value = CreateDkmClrValue("a" & vbNullChar & "b", type:=stringType)
+            result = FormatResult("s", value, inspectionContext:=inspectionContext)
+            Verify(result,
+                   EvalResult("s", "a" & vbNullChar & "b", "String", "s", editableValue:="""a"" & vbNullChar & ""b""", flags:=DkmEvaluationResultFlags.RawString))
+
             ' " " with alias
             value = CreateDkmClrValue(" ", type:=stringType, [alias]:="1", evalFlags:=DkmEvaluationResultFlags.HasObjectId)
             result = FormatResult("s", value, inspectionContext:=inspectionContext)
@@ -67,7 +79,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ExpressionEvaluator
             Dim value = CreateDkmClrValue(ChrW(0), type:=charType)
             Dim result = FormatResult("c", value, inspectionContext:=inspectionContext)
             Verify(result,
-                   EvalResult("c", "vbNullChar", "Char", "c", editableValue:="vbNullChar", flags:=DkmEvaluationResultFlags.None))
+                   EvalResult("c", vbNullChar, "Char", "c", editableValue:="vbNullChar", flags:=DkmEvaluationResultFlags.None))
 
             ' "'"c
             value = CreateDkmClrValue("'"c, type:=charType)
@@ -81,6 +93,24 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.ExpressionEvaluator
             Verify(result,
                    EvalResult("c", """"c, "Char", "c", editableValue:="""""""""c", flags:=DkmEvaluationResultFlags.None))
 
+            ' "\"c
+            value = CreateDkmClrValue("\"c, type:=charType)
+            result = FormatResult("c", value, inspectionContext:=inspectionContext)
+            Verify(result,
+                   EvalResult("c", "\"c, "Char", "c", editableValue:="""\""c", flags:=DkmEvaluationResultFlags.None))
+
+            ' vbLf
+            value = CreateDkmClrValue(ChrW(10), type:=charType)
+            result = FormatResult("c", value, inspectionContext:=inspectionContext)
+            Verify(result,
+                   EvalResult("c", vbLf, "Char", "c", editableValue:="vbLf", flags:=DkmEvaluationResultFlags.None))
+
+            ' ChrW(&H001E)
+            value = CreateDkmClrValue(ChrW(&H001E), type:=charType)
+            result = FormatResult("c", value, inspectionContext:=inspectionContext)
+            Verify(result,
+                   EvalResult("c", New String({ChrW(&H001E)}), "Char", "c", editableValue:="ChrW(30)", flags:=DkmEvaluationResultFlags.None))
+
             ' array
             value = CreateDkmClrValue({"1"c}, type:=charType.MakeArrayType())
             result = FormatResult("a", value, inspectionContext:=inspectionContext)

src/Scripting/VisualBasic/VisualBasicObjectFormatter.vb

@@ -3,13 +3,8 @@
 Imports System
 Imports System.Collections
 Imports System.Collections.Generic
-Imports System.Globalization
 Imports System.Text
-Imports Microsoft.CodeAnalysis
 Imports Microsoft.CodeAnalysis.VisualBasic
-Imports Microsoft.CodeAnalysis.VisualBasic.Symbols
-Imports Microsoft.CodeAnalysis.VisualBasic.Syntax
-Imports Roslyn.Utilities
 
 Namespace Microsoft.CodeAnalysis.Scripting.VisualBasic
 
@@ -50,7 +45,7 @@ Namespace Microsoft.CodeAnalysis.Scripting.VisualBasic
             If useHexadecimalNumbers Then
                 options = options Or ObjectDisplayOptions.UseHexadecimalNumbers
             End If
-            Return ObjectDisplay.FormatLiteral(value, options, nonPrintableSubstitute:=If(quote, Nothing, " "c))
+            Return ObjectDisplay.FormatLiteral(value, options)
         End Function
 
         Public Overrides Function FormatLiteral(c As Char, quote As Boolean, Optional includeCodePoints As Boolean = False, Optional useHexadecimalNumbers As Boolean = False) As String