Merge pull request #6380 from agocke/AddDefaultEncodingRegressionTest

Add regression tests for #4264.

src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs

@@ -91,6 +91,47 @@ private static CSharpCommandLineArguments FullParse(string commandLine, string b
             return CSharpCommandLineParser.Default.Parse(args, baseDirectory, sdkDirectory, additionalReferenceDirectories);
         }
 
+        // This test should only run when the machine's default encoding is shift-JIS
+        [ConditionalFact(typeof(HasShiftJisDefaultEncoding))]
+        public void CompileShiftJisOnShiftJis()
+        {
+            var dir = Temp.CreateDirectory();
+            var src = dir.CreateFile("sjis.cs").WriteAllBytes(TestResources.General.ShiftJisSource);
+
+            var cmd = new MockCSharpCompiler(null, dir.Path, new[] { "/nologo", src.Path });
+
+            Assert.Null(cmd.Arguments.Encoding);
+
+            var outWriter = new StringWriter(CultureInfo.InvariantCulture);
+            var exitCode = cmd.Run(outWriter);
+            Assert.Equal(0, exitCode);
+            Assert.Equal("", outWriter.ToString());
+
+            var result = ProcessUtilities.Run(Path.Combine(dir.Path, "sjis.exe"), arguments: "", workingDirectory: dir.Path);
+            Assert.Equal(0, result.ExitCode);
+            Assert.Equal("星野 八郎太", File.ReadAllText(Path.Combine(dir.Path, "output.txt"), Encoding.GetEncoding(932)));
+        }
+
+        [Fact]
+        public void RunWithShiftJisFile()
+        {
+            var dir = Temp.CreateDirectory();
+            var src = dir.CreateFile("sjis.cs").WriteAllBytes(TestResources.General.ShiftJisSource);
+
+            var cmd = new MockCSharpCompiler(null, dir.Path, new[] { "/nologo", "/codepage:932", src.Path });
+
+            Assert.Equal(932, cmd.Arguments.Encoding?.WindowsCodePage);
+
+            var outWriter = new StringWriter(CultureInfo.InvariantCulture);
+            var exitCode = cmd.Run(outWriter);
+            Assert.Equal(0, exitCode);
+            Assert.Equal("", outWriter.ToString());
+
+            var result = ProcessUtilities.Run(Path.Combine(dir.Path, "sjis.exe"), arguments: "", workingDirectory: dir.Path);
+            Assert.Equal(0, result.ExitCode);
+            Assert.Equal("星野 八郎太", File.ReadAllText(Path.Combine(dir.Path, "output.txt"), Encoding.GetEncoding(932)));
+        }
+
         [Fact]
         [WorkItem(946954)]
         public void CompilerBinariesAreAnyCPU()

src/Compilers/Core/CodeAnalysisTest/Text/EncodedStringTextTests.cs

@@ -16,6 +16,11 @@ private static SourceText CreateMemoryStreamBasedEncodedText(string text, Encodi
         {
             byte[] bytes = writeEncoding.GetBytesWithPreamble(text);
 
+            return CreateMemoryStreamBasedEncodedText(bytes, readEncodingOpt, algorithm);
+        }
+
+        private static SourceText CreateMemoryStreamBasedEncodedText(byte[] bytes, Encoding readEncodingOpt, SourceHashAlgorithm algorithm = SourceHashAlgorithm.Sha1)
+        {
             // For testing purposes, create a bigger buffer so that we verify 
             // that the implementation only uses the part that's covered by the stream and not the entire array.
             byte[] buffer = new byte[bytes.Length + 10];
@@ -27,6 +32,42 @@ private static SourceText CreateMemoryStreamBasedEncodedText(string text, Encodi
             }
         }
 
+        private static SourceText CreateMemoryStreamBasedEncodedText(byte[] bytes, 
+            Func<Encoding> getEncoding,
+            Encoding readEncodingOpt = null,
+            SourceHashAlgorithm algorithm = SourceHashAlgorithm.Sha1)
+        {
+            // For testing purposes, create a bigger buffer so that we verify 
+            // that the implementation only uses the part that's covered by the stream and not the entire array.
+            byte[] buffer = new byte[bytes.Length + 10];
+            bytes.CopyTo(buffer, 0);
+
+            using (var stream = new MemoryStream(buffer, 0, bytes.Length, writable: true, publiclyVisible: true))
+            {
+                return EncodedStringText.Create(stream, getEncoding, readEncodingOpt, algorithm);
+            }
+        }
+
+        [Fact]
+        public void ShiftJisGetEncoding()
+        {
+            var sjis = Encoding.GetEncoding(932);
+            var data = CreateMemoryStreamBasedEncodedText(TestResources.General.ShiftJisSource, () => sjis);
+
+            Assert.Equal(932, data.Encoding?.WindowsCodePage);
+            Assert.Equal(sjis.GetString(TestResources.General.ShiftJisSource), data.ToString());
+        }
+
+        [Fact]
+        public void ShiftJisFile()
+        {
+            var sjis = Encoding.GetEncoding(932);
+            var data = CreateMemoryStreamBasedEncodedText(TestResources.General.ShiftJisSource, sjis);
+
+            Assert.Equal(932, data.Encoding?.WindowsCodePage);
+            Assert.Equal(sjis.GetString(TestResources.General.ShiftJisSource), data.ToString());
+        }
+
         [Fact]
         public void CheckSum002()
         {

src/Compilers/Core/Portable/EncodedStringText.cs

@@ -21,8 +21,8 @@ internal static class EncodedStringText
         /// <summary>
         /// Encoding to use when UTF-8 fails. We try to find the following, in order, if available:
         ///     1. The default ANSI codepage
-       ///      2. CodePage 1252.
-       ///      3. Latin1.
+        ///     2. CodePage 1252.
+        ///     3. Latin1.
         /// </summary>
         private static readonly Encoding s_fallbackEncoding = GetFallbackEncoding();
 
@@ -67,7 +67,20 @@ private static Encoding GetFallbackEncoding()
         /// <paramref name="defaultEncoding"/> is null and the stream appears to be a binary file.
         /// </exception>
         /// <exception cref="IOException">An IO error occurred while reading from the stream.</exception>
-        internal static SourceText Create(Stream stream, Encoding defaultEncoding = null, SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1)
+        internal static SourceText Create(Stream stream,
+            Encoding defaultEncoding = null,
+            SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1)
+        {
+            return Create(stream,
+                () => s_fallbackEncoding,
+                defaultEncoding: defaultEncoding,
+                checksumAlgorithm: checksumAlgorithm);
+        }
+
+        // internal for testing
+        internal static SourceText Create(Stream stream, Func<Encoding> getEncoding,
+            Encoding defaultEncoding = null,
+            SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1)
         {
             Debug.Assert(stream != null);
             Debug.Assert(stream.CanRead && stream.CanSeek);
@@ -87,12 +100,13 @@ internal static SourceText Create(Stream stream, Encoding defaultEncoding = null
 
             try
             {
-                return Decode(stream, defaultEncoding ?? s_fallbackEncoding, checksumAlgorithm, throwIfBinaryDetected: detectEncoding);
+                return Decode(stream, defaultEncoding ?? getEncoding(), checksumAlgorithm, throwIfBinaryDetected: detectEncoding);
             }
             catch (DecoderFallbackException e)
             {
                 throw new InvalidDataException(e.Message);
             }
+
         }
 
         /// <summary>

src/Compilers/Test/Resources/Core/CompilerTestResources.csproj

@@ -83,6 +83,7 @@
     <EmbeddedResource Include="DiagnosticTests\ErrTestMod01.netmodule" />
     <Content Include="DiagnosticTests\ErrTestMod02.cs" />
     <EmbeddedResource Include="DiagnosticTests\ErrTestMod02.netmodule" />
+    <EmbeddedResource Include="Encoding\sjis.cs" />
     <Content Include="MetadataTests\InterfaceAndClass\CSClasses01.cs" />
     <EmbeddedResource Include="MetadataTests\InterfaceAndClass\CSClasses01.dll" />
     <Content Include="MetadataTests\InterfaceAndClass\CSInterfaces01.cs" />

src/Compilers/Test/Resources/Core/Encoding/sjis.cs

+using System.IO;
+using System.Text;
+
+class vleX
+{
+    public static void Main()
+    {
+        File.WriteAllText("output.txt", " Y", Encoding.GetEncoding(932));
+    }
+}

src/Compilers/Test/Resources/Core/TestResources.cs

@@ -178,6 +178,9 @@ public static class General
         private static byte[] _DelegatesWithoutInvoke;
         public static byte[] DelegatesWithoutInvoke => ResourceLoader.GetOrCreateResource(ref _DelegatesWithoutInvoke, "SymbolsTests.Delegates.DelegatesWithoutInvoke.dll");
 
+        private static byte[] _shiftJisSource;
+        public static byte[] ShiftJisSource => ResourceLoader.GetOrCreateResource(ref _shiftJisSource, "Encoding.sjis.cs");
+
         private static byte[] _Events;
         public static byte[] Events => ResourceLoader.GetOrCreateResource(ref _Events, "SymbolsTests.Events.dll");
 

src/Test/Utilities/Shared/Assert/ConditionalFactAttribute.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
 using System;
+using System.Text;
 using Xunit;
 
 namespace Roslyn.Test.Utilities
@@ -29,4 +30,11 @@ public class x86 : ExecutionCondition
 
         public override string SkipReason { get { return "Target platform is not x86"; } }
     }
+
+    public class HasShiftJisDefaultEncoding : ExecutionCondition
+    {
+        public override bool ShouldSkip => Encoding.GetEncoding(0)?.CodePage != 932;
+
+        public override string SkipReason => "OS default codepage is not Shift-JIS (932).";
+    }
 }