Merge pull request #1268 from cston/1138095

Use Unicode escape sequences for control characters

src/Compilers/CSharp/Portable/SymbolDisplay/ObjectDisplay.cs

@@ -133,20 +133,37 @@ internal static string FormatLiteral(bool value)
             return value ? "true" : "false";
         }
 
-        private static void FormatStringChar(
-            ref PooledStringBuilder pooledBuilder,
-            string str,
-            int index,
-            char c,
-            char quote,
-            bool useLanguageSpecificEscapes,
-            bool useUnicodeEscapes)
+        private static void FormatStringChar(StringBuilder builder, char c, char quote)
         {
-            Debug.Assert(quote == '\0' || quote == '"' || quote == '\'');
+            string replaceWith;
+            if (ReplaceChar(c, quote, out replaceWith))
+            {
+                if (replaceWith != null)
+                {
+                    builder.Append(replaceWith);
+                }
+                else
+                {
+                    builder.Append("\\u");
+                    builder.Append(((int)c).ToString("x4"));
+                }
+            }
+            else
+            {
+                builder.Append(c);
+            }
+        }
 
-            string replaceWith = null;
-            if (useLanguageSpecificEscapes)
+        /// <summary>
+        /// Returns true if the character should be replaced and sets
+        /// <paramref name="replaceWith"/> to the replacement text if the
+        /// character is replaced with text other than the Unicode escape sequence.
+        /// </summary>
+        private static bool ReplaceChar(char c, char quote, out string replaceWith)
         {
+            Debug.Assert(quote == '\0' || quote == '"' || quote == '\'');
+
+            replaceWith = null;
             switch (c)
             {
                 case '\\':
@@ -189,47 +206,34 @@ internal static string FormatLiteral(bool value)
                     replaceWith = "\\v";
                     break;
             }
-            }
 
-            bool unicodeEscape = false;
-            if ((replaceWith == null) && useUnicodeEscapes)
+            if (replaceWith != null)
             {
+                return true;
+            }
+
             switch (CharUnicodeInfo.GetUnicodeCategory(c))
             {
                 case UnicodeCategory.Control:
                 case UnicodeCategory.OtherNotAssigned:
                 case UnicodeCategory.ParagraphSeparator:
-                        unicodeEscape = true;
-                        break;
+                    return true;
+                default:
+                    return false;
             }
         }
 
-            if ((replaceWith != null) || unicodeEscape)
-            {
-                if (pooledBuilder == null)
+        private static bool ReplaceAny(string s, char quote)
         {
-                    pooledBuilder = PooledStringBuilder.GetInstance();
-                    if (index > 0)
+            foreach (var c in s)
             {
-                        pooledBuilder.Builder.Append(str, 0, index);
-                    }
-                }
-                var builder = pooledBuilder.Builder;
-                if (replaceWith != null)
-                {
-                    builder.Append(replaceWith);
-                }
-                else
+                string replaceWith;
+                if (ReplaceChar(c, quote, out replaceWith))
                 {
-                    builder.Append("\\u");
-                    builder.Append(((int)c).ToString("x4"));
-                }
+                    return true;
                 }
-            else if (pooledBuilder != null)
-            {
-                var builder = pooledBuilder.Builder;
-                builder.Append(c);
             }
+            return false;
         }
 
         /// <summary>
@@ -252,23 +256,26 @@ public static string FormatLiteral(string value, ObjectDisplayOptions options)
 
             var useQuotes = options.IncludesOption(ObjectDisplayOptions.UseQuotes);
             var quote = useQuotes ? '"' : '\0';
-            PooledStringBuilder pooledBuilder = null;
-            StringBuilder builder = null;
+            if (!useQuotes && !ReplaceAny(value, quote))
+            {
+                return value;
+            }
+
+            var pooledBuilder = PooledStringBuilder.GetInstance();
+            var builder = pooledBuilder.Builder;
             if (useQuotes)
             {
-                pooledBuilder = PooledStringBuilder.GetInstance();
-                builder = pooledBuilder.Builder;
                 builder.Append(quote);
             }
-            for (int i = 0; i < value.Length; i++)
+            foreach (var c in value)
             {
-                FormatStringChar(ref pooledBuilder, value, i, value[i], quote, useLanguageSpecificEscapes: true, useUnicodeEscapes: true);
+                FormatStringChar(builder, c, quote);
             }
             if (useQuotes)
             {
                 builder.Append(quote);
             }
-            return (pooledBuilder == null) ? value : pooledBuilder.ToStringAndFree();
+            return pooledBuilder.ToStringAndFree();
         }
 
         internal static string FormatString(string str, bool useQuotes)
@@ -282,9 +289,9 @@ internal static string FormatString(string str, bool useQuotes)
             var builder = pooledBuilder.Builder;
             const char quote = '"';
             builder.Append(quote);
-            for (int i = 0; i < str.Length; i++)
+            foreach (var c in str)
             {
-                FormatStringChar(ref pooledBuilder, str, i, str[i], quote, useLanguageSpecificEscapes: useQuotes, useUnicodeEscapes: false);
+                FormatStringChar(builder, c, quote);
             }
             builder.Append(quote);
             return pooledBuilder.ToStringAndFree();
@@ -298,24 +305,23 @@ internal static string FormatString(string str, bool useQuotes)
         /// <returns>A character literal with the given value.</returns>
         internal static string FormatLiteral(char c, ObjectDisplayOptions options)
         {
-            var useQuotes = options.IncludesOption(ObjectDisplayOptions.UseQuotes);
-            var quote = useQuotes ? '\'' : '\0';
-            var includeCodePoints = options.IncludesOption(ObjectDisplayOptions.IncludeCodePoints);
             var pooledBuilder = PooledStringBuilder.GetInstance();
             var builder = pooledBuilder.Builder;
-            if (includeCodePoints)
+            if (options.IncludesOption(ObjectDisplayOptions.IncludeCodePoints))
             {
                 builder.Append(options.IncludesOption(ObjectDisplayOptions.UseHexadecimalNumbers) ? "0x" + ((int)c).ToString("x4") : ((int)c).ToString());
                 builder.Append(" ");
             }
-            if (useQuotes)
+            if (options.IncludesOption(ObjectDisplayOptions.UseQuotes))
             {
+                const char quote = '\'';
+                builder.Append(quote);
+                FormatStringChar(builder, c, quote);
                 builder.Append(quote);
             }
-            FormatStringChar(ref pooledBuilder, str: null, index: 0, c: c, quote: quote, useLanguageSpecificEscapes: useQuotes, useUnicodeEscapes: !includeCodePoints);
-            if (useQuotes)
+            else
             {
-                builder.Append(quote);
+                builder.Append(c);
             }
             return pooledBuilder.ToStringAndFree();
         }

src/Compilers/CSharp/Test/Symbol/SymbolDisplay/ObjectDisplayTests.cs

@@ -113,7 +113,7 @@ public void Characters()
 
             Assert.Equal("39 '\\''", ObjectDisplay.FormatLiteral('\'', ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.IncludeCodePoints));
             Assert.Equal("39 '", ObjectDisplay.FormatLiteral('\'', ObjectDisplayOptions.IncludeCodePoints));
-            Assert.Equal("0x001e '\u001e'", ObjectDisplay.FormatLiteral('\u001e', ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.IncludeCodePoints | ObjectDisplayOptions.UseHexadecimalNumbers));
+            Assert.Equal("0x001e '\\u001e'", ObjectDisplay.FormatLiteral('\u001e', ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.IncludeCodePoints | ObjectDisplayOptions.UseHexadecimalNumbers));
             Assert.Equal("0x001e \u001e", ObjectDisplay.FormatLiteral('\u001e', ObjectDisplayOptions.IncludeCodePoints | ObjectDisplayOptions.UseHexadecimalNumbers));
 
             Assert.Equal("0x0008 '\\b'", ObjectDisplay.FormatLiteral('\b', ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.IncludeCodePoints | ObjectDisplayOptions.UseHexadecimalNumbers));
@@ -157,12 +157,12 @@ public void Strings()
 
             // Formatting with quotes should escape specific control characters.
             var expected =
-                "\"\\0\u0001\u0002\u0003\u0004\u0005\u0006\\a\\b\\t\\n\\v\\f\\r\u000e\u000f\u0010" +
-                "\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d" +
-                "\u001e\u001f !\\\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[" +
-                "\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f\u0080\u0081\u0082\u0083\u0084\u0085" +
-                "\u0086\u0087\u0088\u0089\u008a\u008b\u008c\u008d\u008e\u008f\u0090\u0091\u0092" +
-                "\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009a\u009b\u009c\u009d\u009e\u009f" +
+                "\"\\0\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\a\\b\\t\\n\\v\\f\\r\\u000e\\u000f\\u0010" +
+                "\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018\\u0019\\u001a\\u001b\\u001c\\u001d" +
+                "\\u001e\\u001f !\\\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[" +
+                "\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007f\\u0080\\u0081\\u0082\\u0083\\u0084\\u0085" +
+                "\\u0086\\u0087\\u0088\\u0089\\u008a\\u008b\\u008c\\u008d\\u008e\\u008f\\u0090\\u0091\\u0092" +
+                "\\u0093\\u0094\\u0095\\u0096\\u0097\\u0098\\u0099\\u009a\\u009b\\u009c\\u009d\\u009e\\u009f" +
                 " ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèé" +
                 "êëìíîïðñòóôõö÷øùúûüýþ\"";
             Assert.Equal(

src/ExpressionEvaluator/CSharp/Test/ResultProvider/ExpansionTests.cs

@@ -1679,7 +1679,7 @@ public void UnicodeChar()
             value = CreateDkmClrValue('\u001f');
             evalResult = FormatResult("c", value, inspectionContext: CreateDkmInspectionContext(radix: 16));
             Verify(evalResult,
-                EvalResult("c", "0x001f '\u001f'", "char", "c", editableValue: "'\\u001f'"));
+                EvalResult("c", "0x001f '\\u001f'", "char", "c", editableValue: "'\\u001f'"));
 
             // This char is not printable, but there is a specific escape character.
             value = CreateDkmClrValue('\u0007');
@@ -1688,6 +1688,16 @@ public void UnicodeChar()
                 EvalResult("c", "0x0007 '\\a'", "char", "c", editableValue: "'\\a'"));
         }
 
+        [WorkItem(1138095)]
+        [Fact]
+        public void UnicodeString()
+        {
+            var value = CreateDkmClrValue("\u1234\u001f\u0007");
+            var evalResult = FormatResult("s", value);
+            Verify(evalResult,
+                EvalResult("s", $"\"{'\u1234'}\\u001f\\a\"", "string", "s", editableValue: $"\"{'\u1234'}\\u001f\\a\"", flags: DkmEvaluationResultFlags.RawString));
+        }
+
         [WorkItem(1002381)]
         [Fact]
         public void BaseTypeEditableValue()

src/ExpressionEvaluator/CSharp/Test/ResultProvider/FormatSpecifierTests.cs

@@ -50,7 +50,7 @@ public void NoQuotes_String()
             value = CreateDkmClrValue("a\r\n\tb\v\b\u001ec", type: stringType);
             evalResult = FormatResult("s", value, inspectionContext: inspectionContext);
             Verify(evalResult,
-                EvalResult("s", "a\r\n\tb\v\b\u001ec", "string", "s", editableValue: "\"a\\r\\n\\tb\\v\\b\u001ec\"", flags: DkmEvaluationResultFlags.RawString));
+                EvalResult("s", "a\r\n\tb\v\b\u001ec", "string", "s", editableValue: "\"a\\r\\n\\tb\\v\\b\\u001ec\"", flags: DkmEvaluationResultFlags.RawString));
 
             // "a\0b"
             value = CreateDkmClrValue("a\0b", type: stringType);
@@ -58,6 +58,12 @@ public void NoQuotes_String()
             Verify(evalResult,
                 EvalResult("s", "a\0b", "string", "s", editableValue: "\"a\\0b\"", flags: DkmEvaluationResultFlags.RawString));
 
+            // "\u007f\u009f"
+            value = CreateDkmClrValue("\u007f\u009f", type: stringType);
+            evalResult = FormatResult("s", value, inspectionContext: inspectionContext);
+            Verify(evalResult,
+                EvalResult("s", "\u007f\u009f", "string", "s", editableValue: "\"\\u007f\\u009f\"", flags: DkmEvaluationResultFlags.RawString));
+
             // " " with alias
             value = CreateDkmClrValue(" ", type: stringType, alias: "1", evalFlags: DkmEvaluationResultFlags.HasObjectId);
             evalResult = FormatResult("s", value, inspectionContext: inspectionContext);
@@ -118,6 +124,12 @@ public void NoQuotes_Char()
             Verify(evalResult,
                 EvalResult("c", "30 \u001e", "char", "c", editableValue: "'\\u001e'", flags: DkmEvaluationResultFlags.None));
 
+            // '\u007f'
+            value = CreateDkmClrValue('\u007f', type: charType);
+            evalResult = FormatResult("c", value, inspectionContext: inspectionContext);
+            Verify(evalResult,
+                EvalResult("c", "127 \u007f", "char", "c", editableValue: "'\\u007f'", flags: DkmEvaluationResultFlags.None));
+
             // array
             value = CreateDkmClrValue(new char[] { '1' }, type: charType.MakeArrayType());
             evalResult = FormatResult("a", value, inspectionContext: inspectionContext);

src/ExpressionEvaluator/CSharp/Test/ResultProvider/ValueFormattingTests.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
 using System;
+using System.Globalization;
 using Microsoft.CodeAnalysis.ExpressionEvaluator;
 using Microsoft.VisualStudio.Debugger.Evaluation;
 using Xunit;
@@ -83,7 +84,8 @@ public void Char()
             // as a few double-byte characters.  Testing all possible characters takes too long.
             const string format = "{0} '{1}'";
             const string formatUsingHex = "0x{0:x4} '{1}'";
-            for (char ch = (char)0; ch < 0xff; ch++)
+            char ch;
+            for (ch = (char)0; ch < 0xff; ch++)
             {
                 string expected;
                 switch (ch)
@@ -119,21 +121,28 @@ public void Char()
                         expected = "\\\\";
                         break;
                     default:
-                        expected = ch.ToString();
+                        expected = FormatStringChar(ch);
                         break;
                 }
                 Assert.Equal(string.Format(format, (int)ch, expected), FormatValue(ch));
                 Assert.Equal(string.Format(formatUsingHex, (int)ch, expected), FormatValue(ch, useHexadecimal: true));
             }
 
-            foreach (char ch in new[] { (char)0xabcd, (char)0xfeef, (char)0xffef })
-            {
+            ch = (char)0xabcd;
             Assert.Equal(string.Format(format, (int)ch, ch), FormatValue(ch));
             Assert.Equal(string.Format(formatUsingHex, (int)ch, ch), FormatValue(ch, useHexadecimal: true));
-            }
 
-            Assert.Equal("65535 '\uffff'", FormatValue(char.MaxValue));
-            Assert.Equal("0xffff '\uffff'", FormatValue(char.MaxValue, useHexadecimal: true));
+            ch = (char)0xfeef;
+            Assert.Equal(string.Format(format, (int)ch, ch), FormatValue(ch));
+            Assert.Equal(string.Format(formatUsingHex, (int)ch, ch), FormatValue(ch, useHexadecimal: true));
+
+            ch = (char)0xffef;
+            Assert.Equal("65519 '\\uffef'", FormatValue(ch));
+            Assert.Equal("0xffef '\\uffef'", FormatValue(ch, useHexadecimal: true));
+
+            ch = char.MaxValue;
+            Assert.Equal("65535 '\\uffff'", FormatValue(ch));
+            Assert.Equal("0xffff '\\uffff'", FormatValue(ch, useHexadecimal: true));
         }
 
         [Fact]
@@ -181,27 +190,41 @@ public void String()
                         expected = "\\\\";
                         break;
                     default:
-                        expected = ch.ToString();
+                        expected = FormatStringChar(ch);
                         break;
                 }
                 Assert.Equal(string.Format(format, expected), FormatValue(ch.ToString()));
                 Assert.Equal(string.Format(format, expected), FormatValue(ch.ToString(), useHexadecimal: true));
             }
 
-            foreach (char ch in new[] { (char)0xabcd, (char)0xfeef, (char)0xffef })
-            {
-                Assert.Equal(string.Format(format, ch), FormatValue(ch.ToString()));
-                Assert.Equal(string.Format(format, ch), FormatValue(ch.ToString(), useHexadecimal: true));
-            }
+            var s = ((char)0xabcd).ToString();
+            Assert.Equal(string.Format(format, s), FormatValue(s));
+            Assert.Equal(string.Format(format, s), FormatValue(s, useHexadecimal: true));
+
+            s = ((char)0xfeef).ToString();
+            Assert.Equal(string.Format(format, s), FormatValue(s));
+            Assert.Equal(string.Format(format, s), FormatValue(s, useHexadecimal: true));
+
+            s = ((char)0xffef).ToString();
+            Assert.Equal("\"\\uffef\"", FormatValue(s));
+            Assert.Equal("\"\\uffef\"", FormatValue(s, useHexadecimal: true));
 
-            Assert.Equal("\"\uffff\"", FormatValue(char.MaxValue.ToString()));
-            Assert.Equal("\"\uffff\"", FormatValue(char.MaxValue.ToString(), useHexadecimal: true));
+            s = char.MaxValue.ToString();
+            Assert.Equal("\"\\uffff\"", FormatValue(s));
+            Assert.Equal("\"\\uffff\"", FormatValue(s, useHexadecimal: true));
 
             string multiByte = "\ud83c\udfc8";
             Assert.Equal(string.Format(format, "🏈"), FormatValue(multiByte));
             Assert.Equal(string.Format(format, "🏈"), FormatValue(multiByte, useHexadecimal: true));
         }
 
+        private static string FormatStringChar(char c)
+        {
+            return (CharUnicodeInfo.GetUnicodeCategory(c) == UnicodeCategory.Control) ?
+                $"\\u{((int)c).ToString("x4")}" :
+                c.ToString();
+        }
+
         [Fact]
         public void Void()
         {

src/ExpressionEvaluator/VisualBasic/Test/ResultProvider/ExpansionTests.vb

@@ -286,6 +286,15 @@ End Class
                 EvalResult("c", quotedChar, "Char", "c", editableValue:=quotedChar))
         End Sub
 
+        <Fact>
+        Public Sub UnicodeString()
+            Const quotedString = """" & ChrW(&H1234) & """ & ChrW(7)"
+            Dim value = CreateDkmClrValue(New String({ChrW(&H1234), ChrW(&H0007)}))
+            Dim result = FormatResult("s", value)
+            Verify(result,
+                EvalResult("s", quotedString, "String", "s", editableValue:=quotedString, flags:=DkmEvaluationResultFlags.RawString))
+        End Sub
+
         <Fact, WorkItem(1002381)>
         Public Sub BaseTypeEditableValue()
             Dim source = "