diff --git a/src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs b/src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs index d31b2b6b82874b47c40fe2039ad45b3c00512567..c4e5f855d7938790999baf940611721c4a9e8ddd 100644 --- a/src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs +++ b/src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs @@ -91,6 +91,47 @@ private static CSharpCommandLineArguments FullParse(string commandLine, string b return CSharpCommandLineParser.Default.Parse(args, baseDirectory, sdkDirectory, additionalReferenceDirectories); } + // This test should only run when the machine's default encoding is shift-JIS + [ConditionalFact(typeof(HasShiftJisDefaultEncoding))] + public void CompileShiftJisOnShiftJis() + { + var dir = Temp.CreateDirectory(); + var src = dir.CreateFile("sjis.cs").WriteAllBytes(TestResources.General.ShiftJisSource); + + var cmd = new MockCSharpCompiler(null, dir.Path, new[] { "/nologo", src.Path }); + + Assert.Null(cmd.Arguments.Encoding); + + var outWriter = new StringWriter(CultureInfo.InvariantCulture); + var exitCode = cmd.Run(outWriter); + Assert.Equal(0, exitCode); + Assert.Equal("", outWriter.ToString()); + + var result = ProcessUtilities.Run(Path.Combine(dir.Path, "sjis.exe"), arguments: "", workingDirectory: dir.Path); + Assert.Equal(0, result.ExitCode); + Assert.Equal("星野 八郎太", File.ReadAllText(Path.Combine(dir.Path, "output.txt"), Encoding.GetEncoding(932))); + } + + [Fact] + public void RunWithShiftJisFile() + { + var dir = Temp.CreateDirectory(); + var src = dir.CreateFile("sjis.cs").WriteAllBytes(TestResources.General.ShiftJisSource); + + var cmd = new MockCSharpCompiler(null, dir.Path, new[] { "/nologo", "/codepage:932", src.Path }); + + Assert.Equal(932, cmd.Arguments.Encoding?.WindowsCodePage); + + var outWriter = new StringWriter(CultureInfo.InvariantCulture); + var exitCode = cmd.Run(outWriter); + Assert.Equal(0, exitCode); + Assert.Equal("", outWriter.ToString()); + + var result = ProcessUtilities.Run(Path.Combine(dir.Path, "sjis.exe"), arguments: "", workingDirectory: dir.Path); + Assert.Equal(0, result.ExitCode); + Assert.Equal("星野 八郎太", File.ReadAllText(Path.Combine(dir.Path, "output.txt"), Encoding.GetEncoding(932))); + } + [Fact] [WorkItem(946954)] public void CompilerBinariesAreAnyCPU() diff --git a/src/Compilers/Core/CodeAnalysisTest/Text/EncodedStringTextTests.cs b/src/Compilers/Core/CodeAnalysisTest/Text/EncodedStringTextTests.cs index 1e3b1f9585fbe7c84c05cb903b9e655f13e5f32f..9b988b8bbeb29170bc46ffcd01674bf89f532dc1 100644 --- a/src/Compilers/Core/CodeAnalysisTest/Text/EncodedStringTextTests.cs +++ b/src/Compilers/Core/CodeAnalysisTest/Text/EncodedStringTextTests.cs @@ -16,6 +16,11 @@ private static SourceText CreateMemoryStreamBasedEncodedText(string text, Encodi { byte[] bytes = writeEncoding.GetBytesWithPreamble(text); + return CreateMemoryStreamBasedEncodedText(bytes, readEncodingOpt, algorithm); + } + + private static SourceText CreateMemoryStreamBasedEncodedText(byte[] bytes, Encoding readEncodingOpt, SourceHashAlgorithm algorithm = SourceHashAlgorithm.Sha1) + { // For testing purposes, create a bigger buffer so that we verify // that the implementation only uses the part that's covered by the stream and not the entire array. byte[] buffer = new byte[bytes.Length + 10]; @@ -27,6 +32,42 @@ private static SourceText CreateMemoryStreamBasedEncodedText(string text, Encodi } } + private static SourceText CreateMemoryStreamBasedEncodedText(byte[] bytes, + Func getEncoding, + Encoding readEncodingOpt = null, + SourceHashAlgorithm algorithm = SourceHashAlgorithm.Sha1) + { + // For testing purposes, create a bigger buffer so that we verify + // that the implementation only uses the part that's covered by the stream and not the entire array. + byte[] buffer = new byte[bytes.Length + 10]; + bytes.CopyTo(buffer, 0); + + using (var stream = new MemoryStream(buffer, 0, bytes.Length, writable: true, publiclyVisible: true)) + { + return EncodedStringText.Create(stream, getEncoding, readEncodingOpt, algorithm); + } + } + + [Fact] + public void ShiftJisGetEncoding() + { + var sjis = Encoding.GetEncoding(932); + var data = CreateMemoryStreamBasedEncodedText(TestResources.General.ShiftJisSource, () => sjis); + + Assert.Equal(932, data.Encoding?.WindowsCodePage); + Assert.Equal(sjis.GetString(TestResources.General.ShiftJisSource), data.ToString()); + } + + [Fact] + public void ShiftJisFile() + { + var sjis = Encoding.GetEncoding(932); + var data = CreateMemoryStreamBasedEncodedText(TestResources.General.ShiftJisSource, sjis); + + Assert.Equal(932, data.Encoding?.WindowsCodePage); + Assert.Equal(sjis.GetString(TestResources.General.ShiftJisSource), data.ToString()); + } + [Fact] public void CheckSum002() { diff --git a/src/Compilers/Core/Portable/EncodedStringText.cs b/src/Compilers/Core/Portable/EncodedStringText.cs index e2cd5186d4b2165371c805df9e0cf2f029fb9f77..9513504987e1549fa9689db7ec53581b932448fb 100644 --- a/src/Compilers/Core/Portable/EncodedStringText.cs +++ b/src/Compilers/Core/Portable/EncodedStringText.cs @@ -21,8 +21,8 @@ internal static class EncodedStringText /// /// Encoding to use when UTF-8 fails. We try to find the following, in order, if available: /// 1. The default ANSI codepage - /// 2. CodePage 1252. - /// 3. Latin1. + /// 2. CodePage 1252. + /// 3. Latin1. /// private static readonly Encoding s_fallbackEncoding = GetFallbackEncoding(); @@ -67,7 +67,20 @@ private static Encoding GetFallbackEncoding() /// is null and the stream appears to be a binary file. /// /// An IO error occurred while reading from the stream. - internal static SourceText Create(Stream stream, Encoding defaultEncoding = null, SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1) + internal static SourceText Create(Stream stream, + Encoding defaultEncoding = null, + SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1) + { + return Create(stream, + () => s_fallbackEncoding, + defaultEncoding: defaultEncoding, + checksumAlgorithm: checksumAlgorithm); + } + + // internal for testing + internal static SourceText Create(Stream stream, Func getEncoding, + Encoding defaultEncoding = null, + SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1) { Debug.Assert(stream != null); Debug.Assert(stream.CanRead && stream.CanSeek); @@ -87,12 +100,13 @@ internal static SourceText Create(Stream stream, Encoding defaultEncoding = null try { - return Decode(stream, defaultEncoding ?? s_fallbackEncoding, checksumAlgorithm, throwIfBinaryDetected: detectEncoding); + return Decode(stream, defaultEncoding ?? getEncoding(), checksumAlgorithm, throwIfBinaryDetected: detectEncoding); } catch (DecoderFallbackException e) { throw new InvalidDataException(e.Message); } + } /// diff --git a/src/Compilers/Test/Resources/Core/CompilerTestResources.csproj b/src/Compilers/Test/Resources/Core/CompilerTestResources.csproj index 983b617d63cb24d423b4dac3609c7acbd1adcd9e..e194d526d7df0faaff3c861b5686659c4d0f56bb 100644 --- a/src/Compilers/Test/Resources/Core/CompilerTestResources.csproj +++ b/src/Compilers/Test/Resources/Core/CompilerTestResources.csproj @@ -83,6 +83,7 @@ + diff --git a/src/Compilers/Test/Resources/Core/Encoding/sjis.cs b/src/Compilers/Test/Resources/Core/Encoding/sjis.cs new file mode 100644 index 0000000000000000000000000000000000000000..94d741d5548dd49aa7524006cb39f4696c4cf95f --- /dev/null +++ b/src/Compilers/Test/Resources/Core/Encoding/sjis.cs @@ -0,0 +1,10 @@ +using System.IO; +using System.Text; + +class vleX +{ + public static void Main() + { + File.WriteAllText("output.txt", " Y", Encoding.GetEncoding(932)); + } +} diff --git a/src/Compilers/Test/Resources/Core/TestResources.cs b/src/Compilers/Test/Resources/Core/TestResources.cs index 3c2356167a9cf32f8253703e899b6fa05a7d14e5..5ea9ad57c89bab83d7d378972b57167178bf2856 100644 --- a/src/Compilers/Test/Resources/Core/TestResources.cs +++ b/src/Compilers/Test/Resources/Core/TestResources.cs @@ -178,6 +178,9 @@ public static class General private static byte[] _DelegatesWithoutInvoke; public static byte[] DelegatesWithoutInvoke => ResourceLoader.GetOrCreateResource(ref _DelegatesWithoutInvoke, "SymbolsTests.Delegates.DelegatesWithoutInvoke.dll"); + private static byte[] _shiftJisSource; + public static byte[] ShiftJisSource => ResourceLoader.GetOrCreateResource(ref _shiftJisSource, "Encoding.sjis.cs"); + private static byte[] _Events; public static byte[] Events => ResourceLoader.GetOrCreateResource(ref _Events, "SymbolsTests.Events.dll"); diff --git a/src/Test/Utilities/Shared/Assert/ConditionalFactAttribute.cs b/src/Test/Utilities/Shared/Assert/ConditionalFactAttribute.cs index 08e2af68ea62f79b954610ce2cb3af43bd2a465e..6325ae196c2057eb87605a1bfb4fdeda1f900c45 100644 --- a/src/Test/Utilities/Shared/Assert/ConditionalFactAttribute.cs +++ b/src/Test/Utilities/Shared/Assert/ConditionalFactAttribute.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Text; using Xunit; namespace Roslyn.Test.Utilities @@ -29,4 +30,11 @@ public class x86 : ExecutionCondition public override string SkipReason { get { return "Target platform is not x86"; } } } + + public class HasShiftJisDefaultEncoding : ExecutionCondition + { + public override bool ShouldSkip => Encoding.GetEncoding(0)?.CodePage != 932; + + public override string SkipReason => "OS default codepage is not Shift-JIS (932)."; + } }