Merge pull request #4513 from agocke/RewriteEncodingDefaultToDetectCoreCLR

Add encoding detection for CoreCLR

src/Compilers/CSharp/CscCore/project.json

@@ -29,6 +29,7 @@
     "System.Security.Cryptography.Hashing.Algorithms": "4.0.0-beta-23109",
     "System.Security.Principal": "4.0.0-beta-23109",
     "System.Text.Encoding": "4.0.10-beta-23109",
+    "System.Text.Encoding.CodePages": "4.0.0-beta-23019",
     "System.Text.Encoding.Extensions": "4.0.10-beta-23109",
     "System.Threading": "4.0.10-beta-23109",
     "System.Threading.Overlapped": "4.0.0-beta-23109",

src/Compilers/Core/Portable/CodeAnalysis.csproj

@@ -43,6 +43,9 @@
     <ErrorReport>prompt</ErrorReport>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="..\..\Helpers\CoreClrShim.cs">
+      <Link>CoreClrShim.cs</Link>
+    </Compile>
     <Compile Include="AdditionalTextFile.cs" />
     <Compile Include="AssemblyReferenceResolver.cs" />
     <Compile Include="AssemblyUtilities.cs" />

src/Compilers/Core/Portable/EncodedStringText.cs

@@ -16,18 +16,36 @@ internal static class EncodedStringText
         /// Encoding to use when there is no byte order mark (BOM) on the stream. This encoder may throw a <see cref="DecoderFallbackException"/>
         /// if the stream contains invalid UTF-8 bytes.
         /// </summary>
-        private static readonly Encoding s_fallbackEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
+        private static readonly Encoding s_utf8Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
 
         /// <summary>
-        /// Encoding to use when UTF-8 fails and default OS ANSI encoding is not available. If available, we use CodePage 1252. If not, we use Latin1.
+        /// Encoding to use when UTF-8 fails. We try to find the following, in order, if available:
+        ///     1. The default ANSI codepage
+       ///      2. CodePage 1252.
+       ///      3. Latin1.
         /// </summary>
-        private static readonly Encoding s_defaultFallbackEncoding = GetDefaultFallbackEncoding();
+        private static readonly Encoding s_fallbackEncoding = GetFallbackEncoding();
 
-        private static Encoding GetDefaultFallbackEncoding()
+        private static Encoding GetFallbackEncoding()
         {
             try
             {
-                return PortableShim.Encoding.GetEncoding(1252);
+                // If we're running on CoreCLR there is no "default" codepage
+                // but we should be able to grab 1252 from System.Text.Encoding.CodePages
+                if (CoreClrShim.IsCoreClr)
+                {
+                    CoreClrShim.Encoding.RegisterProvider(CoreClrShim.CodePagesEncodingProvider.Instance);
+                    // We should now have 1252 from the CodePagesEncodingProvider
+                    return PortableShim.Encoding.GetEncoding(1252);
+                }
+                // If we're running on the desktop framework we should be able
+                // to get the default ANSI code page in the operating system's
+                // regional and language settings,
+                else
+                {
+                    return PortableShim.Encoding.GetEncoding(0)
+                        ?? PortableShim.Encoding.GetEncoding(1252);
+                }
             }
             catch (NotSupportedException)
             {
@@ -35,20 +53,6 @@ private static Encoding GetDefaultFallbackEncoding()
             }
         }
 
-        private static Encoding GetDefaultAnsiEncoding()
-        {
-            try
-            {
-                // To get the encoding associated with the default ANSI code page in the operating system's regional and language settings,
-                // supply a value 0 for the codepage argument 
-                return PortableShim.Encoding.GetEncoding(0) ?? s_defaultFallbackEncoding;
-            }
-            catch 
-            {
-                return s_defaultFallbackEncoding;
-            }
-        }
-
         /// <summary>
         /// Initializes an instance of <see cref="SourceText"/> from the provided stream. This version differs
         /// from <see cref="SourceText.From(Stream, Encoding, SourceHashAlgorithm, bool)"/> in two ways:
@@ -78,7 +82,7 @@ internal static SourceText Create(Stream stream, Encoding defaultEncoding = null
             {
                 try
                 {
-                    return Decode(stream, s_fallbackEncoding, checksumAlgorithm, throwIfBinaryDetected: false);
+                    return Decode(stream, s_utf8Encoding, checksumAlgorithm, throwIfBinaryDetected: false);
                 }
                 catch (DecoderFallbackException)
                 {
@@ -88,7 +92,7 @@ internal static SourceText Create(Stream stream, Encoding defaultEncoding = null
 
             try
             {
-                return Decode(stream, defaultEncoding ?? GetDefaultAnsiEncoding(), checksumAlgorithm, throwIfBinaryDetected: detectEncoding);
+                return Decode(stream, defaultEncoding ?? s_fallbackEncoding, checksumAlgorithm, throwIfBinaryDetected: detectEncoding);
             }
             catch (DecoderFallbackException e)
             {

src/Compilers/Helpers/CoreClrShim.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Reflection;
+using System.Runtime.ExceptionServices;
+
+namespace Roslyn.Utilities
+{
+
+    /// <summary>
+    /// This is a bridge for APIs that are only available on CoreCLR or .NET 4.6
+    /// and NOT on .NET 4.5. The compiler currently targets .NET 4.5 and CoreCLR
+    /// so this shim is necessary for switching on the dependent behavior.
+    /// </summary>
+    internal static class CoreClrShim
+    {
+        internal static bool IsCoreClr { get; } =
+            Type.GetType("System.Runtime.Loader.AssemblyLoadContext, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
+                         throwOnError: false) != null;
+
+        internal static void Initialize()
+        {
+            // This method provides a way to force the static initializers of each type below
+            // to run. This ensures that the static field values will be computed eagerly
+            // rather than lazily on demand. If you add a new nested class below to access API
+            // surface area, be sure to "touch" the Type field here.
+
+            Touch(CodePagesEncodingProvider.Type);
+        }
+
+        private static void Touch(Type type)
+        {
+            // Do nothing.
+        }
+
+        internal static class CodePagesEncodingProvider
+        {
+            internal static readonly Type Type =
+                Type.GetType("System.Text.CodePagesEncodingProvider, System.Text.Encoding.CodePages, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
+                             throwOnError: false);
+
+            private static PropertyInfo s_instance = Type
+                .GetTypeInfo()
+                .GetDeclaredProperty("Instance");
+
+            internal static object Instance => s_instance.GetValue(null);
+        }
+
+        internal static class Encoding
+        {
+            private static readonly MethodInfo s_registerProvider = PortableShim.Encoding.Type
+                .GetTypeInfo()
+                .GetDeclaredMethod("RegisterProvider");
+
+            internal static void RegisterProvider(object provider)
+            {
+                try
+                {
+                    s_registerProvider.Invoke(null, new[] { provider });
+                }
+                catch (TargetInvocationException e)
+                {
+                    ExceptionDispatchInfo.Capture(e.InnerException).Throw();
+                }
+            }
+        }
+    }
+}

src/Compilers/VisualBasic/VbcCore/project.json

@@ -29,6 +29,7 @@
     "System.Security.Cryptography.Hashing.Algorithms": "4.0.0-beta-23109",
     "System.Security.Principal": "4.0.0-beta-23109",
     "System.Text.Encoding": "4.0.10-beta-23109",
+    "System.Text.Encoding.CodePages": "4.0.0-beta-23019",
     "System.Text.Encoding.Extensions": "4.0.10-beta-23109",
     "System.Threading": "4.0.10-beta-23109",
     "System.Threading.Overlapped": "4.0.0-beta-23109",

src/Workspaces/Core/Desktop/Workspaces.Desktop.csproj

@@ -68,6 +68,9 @@
     <Compile Include="..\..\..\Compilers\Core\Portable\EncodedStringText.cs">
       <Link>InternalUtilities\EncodedStringText.cs</Link>
     </Compile>
+    <Compile Include="..\..\..\Compilers\Helpers\CoreClrShim.cs">
+      <Link>InternalUtilities\CoreClrShim.cs</Link>
+    </Compile>
     <Compile Include="..\..\..\Compilers\Core\Portable\FileKey.cs">
       <Link>InternalUtilities\FileKey.cs</Link>
     </Compile>