提交 d198bb44 编写于 作者: T Tarek Mahmoud Sayed 提交者: GitHub

Enable full Encodings on Windows Console by Default (dotnet/corefx#14649)

* Enable full Encodings on Windows Console by Default

by default .NetCore doesn't enable the codepage encoding and the app has to opt-in to the full encoding through the call
	Encoding.RegisterProvider(CodePagesEncodingProvider.Instance)
This was not a good stand for the Console specially on Windows as the Console always using encoding which is not supported by default and we had to fallback using UTF8.
In this change we are supporting the full OS encoding on Console and System.Diagnostics.Process without bringing the big encoding size cost (which is 600+ KB). The way we support it is we create an OSEncoding class wrapper which will call the OS instead of using our own encoding data.
The only restriction we’ll have is cannot use encoding fallback with OSEncoding as this will be very slow and not intuitive. But this is acceptable as usually apps don’t use the console encodings for such fallback operation. But if anyone want to do that can still register the encoding provider and get full support.

* Add Encoder/Decoder to OSEncoding in Console

* fix misspelling

* Some fixes and adding tests

* Fix corner case when buffer has only one high surrogate char


Commit migrated from https://github.com/dotnet/corefx/commit/bb068f3be887ed34e7f8bedc4c12ec4284884441
上级 e467217e
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
internal static partial class Interop
{
internal static partial class Kernel32
{
internal const uint CP_ACP = 0;
internal const uint WC_NO_BEST_FIT_CHARS = 0x00000400;
}
}
......@@ -8,8 +8,6 @@ internal partial class Interop
{
internal partial class Kernel32
{
private const uint CP_ACP = 0x0u;
[DllImport(Libraries.Kernel32, CharSet = CharSet.Unicode, EntryPoint = "GetCPInfoExW")]
private extern static unsafe int GetCPInfoExW(uint CodePage, uint dwFlags, CPINFOEXW* lpCPInfoEx);
......@@ -24,18 +22,37 @@ private unsafe struct CPINFOEXW
internal fixed byte CodePageName[260];
}
internal static unsafe int GetLeadByteRanges(int codePage, byte[] leadyByteRanges)
{
int count = 0;
CPINFOEXW cpInfo;
if (GetCPInfoExW((uint) codePage, 0, &cpInfo) != 0)
{
// we don't care about the last 2 bytes as those are nulls
for (int i=0; i<10; i+=2)
{
if (leadyByteRanges[i] == 0)
break;
leadyByteRanges[i] = cpInfo.LeadByte[i];
leadyByteRanges[i+1] = cpInfo.LeadByte[i+1];
count++;
}
}
return count;
}
internal static bool TryGetACPCodePage(out int codePage)
{
codePage = 0;
// Note: GetACP is not available in the Windows Store Profile, but calling
// GetCPInfoEx with the value CP_ACP (0) yields the same result.
CPINFOEXW _cpInfo;
CPINFOEXW cpInfo;
unsafe
{
CPINFOEXW* _lpCPInfoExPtr = &(_cpInfo);
if (GetCPInfoExW(CP_ACP, 0, _lpCPInfoExPtr) != 0)
CPINFOEXW* lpCPInfoExPtr = &(cpInfo);
if (GetCPInfoExW(CP_ACP, 0, lpCPInfoExPtr) != 0)
{
codePage = (int)_cpInfo.CodePage;
codePage = (int)cpInfo.CodePage;
return true;
}
}
......
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Runtime.InteropServices;
internal partial class Interop
{
internal partial class Kernel32
{
[DllImport(Libraries.Kernel32)]
internal static extern unsafe int MultiByteToWideChar(
uint CodePage, uint dwFlags,
byte* lpMultiByteStr, int cbMultiByte,
char* lpWideCharStr, int cchWideChar);
}
}
......@@ -16,7 +16,5 @@ internal partial class Kernel32
byte* lpMultiByteStr, int cbMultiByte,
IntPtr lpDefaultChar, IntPtr lpUsedDefaultChar);
internal const uint CP_ACP = 0;
internal const uint WC_NO_BEST_FIT_CHARS = 0x00000400;
}
}
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Text;
using System.Diagnostics;
namespace System.Text
{
internal class DecoderDBCS : Decoder
{
private Encoding _encoding;
private byte[] _leadyByteRanges = new byte[10]; // Max 5 ranges
private int _rangesCount;
private byte _leftOverLeadByte;
internal DecoderDBCS(Encoding encoding)
{
_encoding = encoding;
_rangesCount = Interop.Kernel32.GetLeadByteRanges(_encoding.CodePage, _leadyByteRanges);
Reset();
}
private bool IsLeadByte(byte b)
{
if (b < _leadyByteRanges[0])
return false;
int i = 0;
while (i < _rangesCount)
{
if (b >= _leadyByteRanges[i] && b <= _leadyByteRanges[i + 1])
return true;
i += 2;
}
return false;
}
public override void Reset()
{
_leftOverLeadByte = 0;
}
public override unsafe int GetCharCount(byte[] bytes, int index, int count)
{
return GetCharCount(bytes, index, count, false);
}
public override unsafe int GetCharCount(byte[] bytes, int index, int count, bool flush)
{
if (bytes == null)
throw new ArgumentNullException(nameof(bytes), SR.ArgumentNull_Array);
if (index < 0 || count < 0)
throw new ArgumentOutOfRangeException(index < 0 ? nameof(index) : nameof(count), SR.ArgumentOutOfRange_NeedNonNegNum);
if (bytes.Length - index < count)
throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer);
if (count == 0 && (_leftOverLeadByte == 0 || !flush))
return 0;
fixed (byte* pBytes = bytes)
{
byte dummyByte;
byte* pBuffer = pBytes == null ? &dummyByte : pBytes + index;
return GetCharCount(pBuffer, count, flush);
}
}
private unsafe int ConvertWithLeftOverByte(byte* bytes, int count, char* chars, int charCount)
{
Debug.Assert(_leftOverLeadByte != 0);
byte* pTempBuffer = stackalloc byte[2];
pTempBuffer[0] = _leftOverLeadByte;
int index = 0;
if (count > 0)
{
pTempBuffer[1] = bytes[0];
index++;
}
int result = OSEncoding.MultiByteToWideChar(_encoding.CodePage, pTempBuffer, index+1, chars, charCount);
if (count - index > 0)
result += OSEncoding.MultiByteToWideChar(
_encoding.CodePage, bytes + index,
count - index,
chars == null ? null : chars + result,
chars == null ? 0 : charCount - result);
return result;
}
public unsafe override int GetCharCount(byte* bytes, int count, bool flush)
{
if (bytes == null)
throw new ArgumentNullException(nameof(bytes), SR.ArgumentNull_Array);
if (count < 0)
throw new ArgumentOutOfRangeException(nameof(count), SR.ArgumentOutOfRange_NeedNonNegNum);
bool excludeLastByte = count > 0 || (!flush && IsLastByteALeadByte(bytes, count));
if (excludeLastByte)
count--;
if (_leftOverLeadByte == 0)
{
if (count <= 0)
return 0;
return OSEncoding.MultiByteToWideChar(_encoding.CodePage, bytes, count, null, 0);
}
if (count == 0 && !excludeLastByte && !flush)
return 0;
return ConvertWithLeftOverByte(bytes, count, null, 0);
}
public override unsafe int GetChars(byte[] bytes, int byteIndex, int byteCount,
char[] chars, int charIndex)
{
return GetChars(bytes, byteIndex, byteCount, chars, charIndex, false);
}
public override unsafe int GetChars(byte[] bytes, int byteIndex, int byteCount,
char[] chars, int charIndex, bool flush)
{
if (bytes == null || chars == null)
throw new ArgumentNullException(bytes == null ? nameof(bytes) : nameof(chars), SR.ArgumentNull_Array);
if (byteIndex < 0 || byteCount < 0)
throw new ArgumentOutOfRangeException(byteIndex < 0 ? nameof(byteIndex) : nameof(byteCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (bytes.Length - byteIndex < byteCount)
throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer);
if (charIndex < 0 || charIndex > chars.Length)
throw new ArgumentOutOfRangeException(nameof(charIndex), SR.ArgumentOutOfRange_Index);
if (chars.Length == 0)
return 0;
if (byteCount == 0 && (_leftOverLeadByte == 0 || !flush))
return 0;
fixed (char* pChars = chars)
fixed (byte* pBytes = bytes)
{
byte dummyByte;
byte* pBuffer = pBytes == null ? &dummyByte : pBytes + byteIndex;
return GetChars(pBuffer, byteCount, pChars + charIndex, chars.Length - charIndex, flush);
}
}
public unsafe override int GetChars(byte* bytes, int byteCount, char* chars, int charCount, bool flush)
{
if (chars == null || bytes == null)
throw new ArgumentNullException(chars == null ? nameof(chars) : nameof(bytes), SR.ArgumentNull_Array);
if (byteCount < 0 || charCount < 0)
throw new ArgumentOutOfRangeException(byteCount < 0 ? nameof(byteCount) : nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (charCount == 0)
return 0;
byte lastByte = byteCount > 0 && !flush && IsLastByteALeadByte(bytes, byteCount) ? bytes[byteCount - 1] : (byte) 0;
if (lastByte != 0)
byteCount--;
if (_leftOverLeadByte == 0)
{
if (byteCount <= 0)
{
_leftOverLeadByte = lastByte;
return 0;
}
int result = OSEncoding.MultiByteToWideChar(_encoding.CodePage, bytes, byteCount, chars, charCount);
_leftOverLeadByte = lastByte;
return result;
}
// we have left over lead byte
if (byteCount == 0 && lastByte == 0 && !flush)
return 0;
int res = ConvertWithLeftOverByte(bytes, byteCount, chars, charCount);
_leftOverLeadByte = lastByte;
return res;
}
public override unsafe void Convert(byte[] bytes, int byteIndex, int byteCount,
char[] chars, int charIndex, int charCount, bool flush,
out int bytesUsed, out int charsUsed, out bool completed)
{
if (bytes == null || chars == null)
throw new ArgumentNullException(bytes == null ? nameof(bytes) : nameof(chars), SR.ArgumentNull_Array);
if (byteIndex < 0 || byteCount < 0)
throw new ArgumentOutOfRangeException(byteIndex < 0 ? nameof(byteIndex) : nameof(byteCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (charIndex < 0 || charCount < 0)
throw new ArgumentOutOfRangeException(charIndex < 0 ? nameof(charIndex) : nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (bytes.Length - byteIndex < byteCount)
throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer);
if (chars.Length - charIndex < charCount)
throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCountBuffer);
if (charCount == 0 || (bytes.Length == 0 && (_leftOverLeadByte == 0 || !flush)))
{
bytesUsed = 0;
charsUsed = 0;
completed = false;
return;
}
fixed (char* pChars = chars)
fixed (byte* pBytes = bytes)
{
byte dummyByte;
byte* pBuffer = pBytes == null ? &dummyByte : pBytes + byteIndex;
Convert(pBuffer, byteCount, pChars + charIndex, charCount, flush, out bytesUsed, out charsUsed, out completed);
}
}
public unsafe override void Convert(byte* bytes, int byteCount,
char* chars, int charCount, bool flush,
out int bytesUsed, out int charsUsed, out bool completed)
{
if (chars == null || bytes == null)
throw new ArgumentNullException(chars == null ? nameof(chars) : nameof(bytes), SR.ArgumentNull_Array);
if (byteCount < 0 || charCount < 0)
throw new ArgumentOutOfRangeException(byteCount < 0 ? nameof(byteCount) : nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
int count = byteCount;
while (count > 0)
{
int returnedCharCount = GetCharCount(bytes, count, flush);
if (returnedCharCount <= charCount)
break;
count /= 2;
}
if (count > 0)
{
// note GetChars can change the _leftOverLeadByte state
charsUsed = GetChars(bytes, count, chars, charCount, flush);
bytesUsed = count;
completed = _leftOverLeadByte == 0 && byteCount == count;
return;
}
bytesUsed = 0;
charsUsed = 0;
completed = false;
}
// not IsLastByteALeadByte depends on the _leftOverLeadByte state
private unsafe bool IsLastByteALeadByte(byte* bytes, int count)
{
if (!IsLeadByte(bytes[count - 1]))
return false; // no need to process the buffer
int index = 0;
if (_leftOverLeadByte != 0)
index++; // trail byte
while (index < count)
{
if (IsLeadByte(bytes[index]))
{
index++;
if (index >= count)
return true;
}
index++;
}
return false;
}
}
}
......@@ -12,7 +12,7 @@ internal static partial class EncodingHelper
{
// Since only a minimum set of encodings are available by default,
// Console encoding might not be available and require provider registering.
// To avoid encoding exception in Console APIs we default to UTF8 in such scenarios.
// To avoid encoding exception in Console APIs we fallback to OSEncoding.
//
//
// The guaranteed way to identify the above is to use a try/catch pattern, however to avoid 1st chance exceptions
......@@ -30,14 +30,14 @@ internal static Encoding GetSupportedConsoleEncoding(int codepage)
if ((defaultEncCodePage == codepage) || defaultEncCodePage != Encoding.UTF8.CodePage)
{
try
{
return Encoding.GetEncoding(codepage);
}
catch (NotSupportedException)
{
}
return Encoding.GetEncoding(codepage);
}
if (codepage != Encoding.UTF8.CodePage)
{
return new OSEncoding(codepage);
}
return new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
}
}
......
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Text;
using System.Diagnostics;
namespace System.Text
{
internal class OSEncoder : Encoder
{
private char _charLeftOver;
private Encoding _encoding;
private const char NULL_CHAR = (char)0;
internal OSEncoder(Encoding encoding)
{
_encoding = encoding;
Reset();
}
public override void Reset()
{
_charLeftOver = NULL_CHAR;
}
public override unsafe int GetByteCount(char[] chars, int index, int count, bool flush)
{
if (chars == null)
throw new ArgumentNullException(nameof(chars), SR.ArgumentNull_Array);
if (index < 0 || count < 0)
throw new ArgumentOutOfRangeException(index < 0 ? nameof(index) : nameof(count), SR.ArgumentOutOfRange_NeedNonNegNum);
if (chars.Length - index < count)
throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCountBuffer);
if (chars.Length == 0 && (_charLeftOver == NULL_CHAR || !flush))
return 0;
fixed (char* pChar = chars)
{
char dummyChar;
char* pBuffer = pChar == null ? &dummyChar : pChar + index;
return GetByteCount(pBuffer, count, flush);
}
}
private unsafe int ConvertWithLeftOverChar(char* chars, int count, byte* bytes, int byteCount)
{
Debug.Assert(_charLeftOver != NULL_CHAR);
char* pTempBuffer = stackalloc char[2];
pTempBuffer[0] = _charLeftOver;
int index = 0;
if (count > 0 && Char.IsLowSurrogate(chars[0]))
{
pTempBuffer[1] = chars[0];
index++;
}
int result = OSEncoding.WideCharToMultiByte(_encoding.CodePage, pTempBuffer, index+1, bytes, byteCount);
if (count - index > 0)
result += OSEncoding.WideCharToMultiByte(
_encoding.CodePage, chars + index,
count - index,
bytes == null ? null : bytes + result,
bytes == null ? 0 : byteCount - result);
return result;
}
public unsafe override int GetByteCount(char* chars, int count, bool flush)
{
if (chars == null)
throw new ArgumentNullException(nameof(chars), SR.ArgumentNull_Array);
if (count < 0)
throw new ArgumentOutOfRangeException(nameof(count), SR.ArgumentOutOfRange_NeedNonNegNum);
bool excludeLastChar = count > 0 && !flush && Char.IsHighSurrogate(chars[count - 1]);
if (excludeLastChar)
count--;
if (_charLeftOver == NULL_CHAR)
{
if (count <= 0)
return 0;
return OSEncoding.WideCharToMultiByte(_encoding.CodePage, chars, count, null, 0);
}
// we have left over character
if (count == 0 && !excludeLastChar && !flush)
return 0;
return ConvertWithLeftOverChar(chars, count, null, 0);
}
public override unsafe int GetBytes(char[] chars, int charIndex, int charCount,
byte[] bytes, int byteIndex, bool flush)
{
if (chars == null || bytes == null)
throw new ArgumentNullException(chars == null ? nameof(chars) : nameof(bytes), SR.ArgumentNull_Array);
if (charIndex < 0 || charCount < 0)
throw new ArgumentOutOfRangeException(charIndex < 0 ? nameof(charIndex) : nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (chars.Length - charIndex < charCount)
throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCountBuffer);
if (byteIndex < 0 || byteIndex > bytes.Length)
throw new ArgumentOutOfRangeException(nameof(byteIndex), SR.ArgumentOutOfRange_Index);
if (bytes.Length == 0)
return 0;
if (charCount == 0 && (_charLeftOver == NULL_CHAR || !flush))
return 0;
fixed (char* pChars = chars)
fixed (byte* pBytes = bytes)
{
char dummyChar;
char* pBuffer = pChars == null ? &dummyChar : pChars + charIndex;
return GetBytes(pBuffer, charCount, pBytes + byteIndex, bytes.Length - byteIndex, flush);
}
}
public unsafe override int GetBytes(char* chars, int charCount, byte* bytes, int byteCount, bool flush)
{
if (chars == null || bytes == null)
throw new ArgumentNullException(chars == null ? nameof(chars) : nameof(bytes), SR.ArgumentNull_Array);
if (byteCount < 0 || charCount < 0)
throw new ArgumentOutOfRangeException(byteCount < 0 ? nameof(byteCount) : nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (byteCount == 0)
return 0;
char lastChar = charCount > 0 && !flush && Char.IsHighSurrogate(chars[charCount - 1]) ? chars[charCount - 1] : NULL_CHAR;
if (lastChar != NULL_CHAR)
charCount--;
if (_charLeftOver == NULL_CHAR)
{
if (charCount <= 0)
{
_charLeftOver = lastChar;
return 0;
}
int result = OSEncoding.WideCharToMultiByte(_encoding.CodePage, chars, charCount, bytes, byteCount);
_charLeftOver = lastChar;
return result;
}
// we have left over character
if (charCount == 0 && lastChar == NULL_CHAR && !flush)
return 0;
int res = ConvertWithLeftOverChar(chars, charCount, bytes, byteCount);
_charLeftOver = lastChar;
return res;
}
public override unsafe void Convert(char[] chars, int charIndex, int charCount,
byte[] bytes, int byteIndex, int byteCount, bool flush,
out int charsUsed, out int bytesUsed, out bool completed)
{
if (chars == null || bytes == null)
throw new ArgumentNullException(chars == null ? nameof(chars) : nameof(bytes), SR.ArgumentNull_Array);
if (charIndex < 0 || charCount < 0)
throw new ArgumentOutOfRangeException(charIndex < 0 ? nameof(charIndex) : nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (byteIndex < 0 || byteCount < 0)
throw new ArgumentOutOfRangeException(byteIndex < 0 ? nameof(byteIndex) : nameof(byteCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (chars.Length - charIndex < charCount)
throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCountBuffer);
if (bytes.Length - byteIndex < byteCount)
throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer);
if (bytes.Length == 0 || (chars.Length == 0 && (_charLeftOver == NULL_CHAR || !flush)))
{
bytesUsed = 0;
charsUsed = 0;
completed = false;
return;
}
fixed (char* pChars = chars)
fixed (byte* pBytes = bytes)
{
char dummyChar;
char* pBuffer = pChars == null ? &dummyChar : pChars + charIndex;
Convert(pBuffer, charCount, pBytes + byteIndex, byteCount, flush, out charsUsed, out bytesUsed, out completed);
}
}
public override unsafe void Convert(char* chars, int charCount,
byte* bytes, int byteCount, bool flush,
out int charsUsed, out int bytesUsed, out bool completed)
{
if (bytes == null || chars == null)
throw new ArgumentNullException(bytes == null ? nameof(bytes) : nameof(chars), SR.ArgumentNull_Array);
if (charCount < 0 || byteCount < 0)
throw new ArgumentOutOfRangeException(charCount < 0 ? nameof(charCount) : nameof(byteCount), SR.ArgumentOutOfRange_NeedNonNegNum);
int count = charCount;
while (count > 0)
{
int returnedByteCount = GetByteCount(chars, count, flush);
if (returnedByteCount <= byteCount)
break;
count /= 2;
}
if (count > 0)
{
// note GetBytes can change the _charLeftOver state
bytesUsed = GetBytes(chars, count, bytes, byteCount, flush);
charsUsed = count;
completed = _charLeftOver == NULL_CHAR && charCount == count;
return;
}
bytesUsed = 0;
charsUsed = 0;
completed = false;
}
public Encoding Encoding
{
get
{
return _encoding;
}
}
}
}
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Text;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
namespace System.Text
{
internal class OSEncoding : Encoding
{
private int _codePage;
private string _encodingName;
internal OSEncoding(int codePage) : base(codePage)
{
_codePage = codePage;
}
public override unsafe int GetByteCount(char[] chars, int index, int count)
{
if (chars == null)
throw new ArgumentNullException(nameof(chars), SR.ArgumentNull_Array);
if (index < 0 || count < 0)
throw new ArgumentOutOfRangeException(index < 0 ? nameof(index) : nameof(count), SR.ArgumentOutOfRange_NeedNonNegNum);
if (chars.Length - index < count)
throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCountBuffer);
if (count == 0)
return 0;
fixed (char* pChar = chars)
{
return WideCharToMultiByte(_codePage, pChar+index, count, null, 0);
}
}
public override unsafe int GetByteCount(String s)
{
// Validate input
if (s == null)
throw new ArgumentNullException(nameof(s));
if (s.Length == 0)
return 0;
fixed (char* pChars = s)
{
return WideCharToMultiByte(_codePage, pChars, s.Length, null, 0);
}
}
public override unsafe int GetBytes(String s, int charIndex, int charCount, byte[] bytes, int byteIndex)
{
if (s == null || bytes == null)
throw new ArgumentNullException(s == null ? nameof(s) : nameof(bytes), SR.ArgumentNull_Array);
if (charIndex < 0 || charCount < 0)
throw new ArgumentOutOfRangeException(charIndex < 0 ? nameof(charIndex) : nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (s.Length - charIndex < charCount)
throw new ArgumentOutOfRangeException(nameof(s), SR.ArgumentOutOfRange_IndexCount);
if (byteIndex < 0 || byteIndex > bytes.Length)
throw new ArgumentOutOfRangeException(nameof(byteIndex), SR.ArgumentOutOfRange_Index);
if (charCount == 0)
return 0;
if (bytes.Length == 0)
{
throw new ArgumentOutOfRangeException(SR.Argument_EncodingConversionOverflowBytes);
}
fixed (char* pChars = s)
fixed (byte *pBytes = bytes)
{
return WideCharToMultiByte(_codePage, pChars+charIndex, charCount, pBytes+byteIndex, bytes.Length - byteIndex);
}
}
public override unsafe int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex)
{
if (chars == null || bytes == null)
throw new ArgumentNullException(chars == null ? nameof(chars) : nameof(bytes), SR.ArgumentNull_Array);
if (charIndex < 0 || charCount < 0)
throw new ArgumentOutOfRangeException(charIndex < 0 ? nameof(charIndex) : nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (chars.Length - charIndex < charCount)
throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCountBuffer);
if (byteIndex < 0 || byteIndex > bytes.Length)
throw new ArgumentOutOfRangeException(nameof(byteIndex), SR.ArgumentOutOfRange_Index);
if (charCount == 0)
return 0;
if (bytes.Length == 0)
{
throw new ArgumentOutOfRangeException(SR.Argument_EncodingConversionOverflowBytes);
}
fixed (char* pChars = chars)
fixed (byte *pBytes = bytes)
{
return WideCharToMultiByte(_codePage, pChars+charIndex, charCount, pBytes+byteIndex, bytes.Length - byteIndex);
}
}
public override unsafe int GetCharCount(byte[] bytes, int index, int count)
{
if (bytes == null)
throw new ArgumentNullException(nameof(bytes), SR.ArgumentNull_Array);
if (index < 0 || count < 0)
throw new ArgumentOutOfRangeException(index < 0 ? nameof(index) : nameof(count), SR.ArgumentOutOfRange_NeedNonNegNum);
if (bytes.Length - index < count)
throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer);
if (count == 0)
return 0;
fixed (byte* pBytes = bytes)
{
return MultiByteToWideChar(_codePage, pBytes+index, count, null, 0);
}
}
public override unsafe int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex)
{
if (bytes == null || chars == null)
throw new ArgumentNullException(bytes == null ? nameof(bytes) : nameof(chars), SR.ArgumentNull_Array);
if (byteIndex < 0 || byteCount < 0)
throw new ArgumentOutOfRangeException(byteIndex < 0 ? nameof(byteIndex) : nameof(byteCount), SR.ArgumentOutOfRange_NeedNonNegNum);
if (bytes.Length - byteIndex < byteCount)
throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer);
if (charIndex < 0 || charIndex > chars.Length)
throw new ArgumentOutOfRangeException(nameof(charIndex), SR.ArgumentOutOfRange_Index);
if (byteCount == 0)
return 0;
if (chars.Length == 0)
throw new ArgumentOutOfRangeException(SR.Argument_EncodingConversionOverflowChars);
fixed (byte* pBytes = bytes)
fixed (char* pChars = chars)
{
return MultiByteToWideChar(_codePage, pBytes+byteIndex, byteCount, pChars+charIndex, chars.Length - charIndex);
}
}
public override int GetMaxByteCount(int charCount)
{
if (charCount < 0)
throw new ArgumentOutOfRangeException(nameof(charCount), SR.ArgumentOutOfRange_NeedNonNegNum);
long byteCount = (long)charCount * 14; // Max possible value for all encodings
if (byteCount > 0x7fffffff)
throw new ArgumentOutOfRangeException(nameof(charCount), SR.ArgumentOutOfRange_GetByteCountOverflow);
return (int)byteCount;
}
public override int GetMaxCharCount(int byteCount)
{
if (byteCount < 0)
throw new ArgumentOutOfRangeException(nameof(byteCount), SR.ArgumentOutOfRange_NeedNonNegNum);
long charCount = byteCount * 4; // Max possible value for all encodings
if (charCount > 0x7fffffff)
throw new ArgumentOutOfRangeException(nameof(byteCount), SR.ArgumentOutOfRange_GetCharCountOverflow);
return (int)charCount;
}
public override String EncodingName
{
get
{
if (_encodingName == null)
{
_encodingName = "Codepage - " + _codePage.ToString();
}
return _encodingName;
}
}
public override String WebName
{
get
{
return EncodingName;
}
}
public override Encoder GetEncoder()
{
return new OSEncoder(this);
}
public override Decoder GetDecoder()
{
switch (CodePage)
{
case 932: // Japanese (Shift-JIS)
case 936: // Chinese Simplified (GB2312)
case 949: // Korean
case 950: // Chinese Traditional (Big5)
case 1361: // Korean (Johab)
case 10001: // Japanese (Mac)
case 10002: // Chinese Traditional (Mac)
case 10003: // Korean (Mac)
case 10008: // Chinese Simplified (Mac)
case 20000: // Chinese Traditional (CNS)
case 20001: // TCA Taiwan
case 20002: // Chinese Traditional (Eten)
case 20003: // IBM5550 Taiwan
case 20004: // TeleText Taiwan
case 20005: // Wang Taiwan
case 20261: // T.61
case 20932: // Japanese (JIS 0208-1990 and 0212-1990)
case 20936: // Chinese Simplified (GB2312-80)
case 51949: // Korean (EUC)
return new DecoderDBCS(this);
default:
return base.GetDecoder();
}
}
internal static unsafe int WideCharToMultiByte(int codePage, char* pChars, int count, byte* pBytes, int byteCount)
{
int result = Interop.Kernel32.WideCharToMultiByte((uint)codePage, 0, pChars, count, pBytes, byteCount, IntPtr.Zero, IntPtr.Zero);
if (result <= 0)
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex);
return result;
}
internal static unsafe int MultiByteToWideChar(int codePage, byte* pBytes, int byteCount, char* pChars, int count)
{
int result = Interop.Kernel32.MultiByteToWideChar((uint)codePage, 0, pBytes, byteCount, pChars, count);
if (result <= 0)
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex);
return result;
}
}
}
\ No newline at end of file
......@@ -234,4 +234,31 @@
<data name="ArgumentOutOfRange_BeepFrequency" xml:space="preserve">
<value>The frequency must be between {0} and {1}.</value>
</data>
<data name="ArgumentNull_Array" xml:space="preserve">
<value>Array cannot be null.</value>
</data>
<data name="ArgumentOutOfRange_IndexCountBuffer" xml:space="preserve">
<value>Index and count must refer to a location within the buffer.</value>
</data>
<data name="ArgumentOutOfRange_IndexCount" xml:space="preserve">
<value>Index and count must refer to a location within the string.</value>
</data>
<data name="ArgumentOutOfRange_Index" xml:space="preserve">
<value>Index was out of range. Must be non-negative and less than the size of the collection.</value>
</data>
<data name="Argument_EncodingConversionOverflowBytes" xml:space="preserve">
<value>The output byte buffer is too small to contain the encoded data, encoding '{0}' fallback '{1}'.</value>
</data>
<data name="Argument_EncodingConversionOverflowChars" xml:space="preserve">
<value>The output char buffer is too small to contain the decoded characters, encoding '{0}' fallback '{1}'.</value>
</data>
<data name="ArgumentOutOfRange_GetByteCountOverflow" xml:space="preserve">
<value>Too many characters. The resulting number of bytes is larger than what can be returned as an int.</value>
</data>
<data name="ArgumentOutOfRange_GetCharCountOverflow" xml:space="preserve">
<value>Too many bytes. The resulting number of chars is larger than what can be returned as an int.</value>
</data>
<data name="Argument_InvalidCharSequenceNoIndex" xml:space="preserve">
<value>String contains invalid Unicode code points.</value>
</data>
</root>
......@@ -37,9 +37,6 @@
<Compile Include="$(CommonPath)\System\Text\ConsoleEncoding.cs">
<Link>Common\System\Text\ConsoleEncoding.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\EncodingHelper.cs">
<Link>Common\System\IO\EncodingHelper.cs</Link>
</Compile>
</ItemGroup>
<!-- Windows : UAP
<ItemGroup Condition="'$(TargetsWindows)' == 'true' And '$(TargetGroup)' == 'uap101aot' ">
......@@ -49,6 +46,21 @@
<!-- Windows : Win32 -->
<ItemGroup Condition="'$(TargetsWindows)' == 'true' And '$(TargetGroup)' == ''">
<Compile Include="System\ConsolePal.Windows.cs" />
<Compile Include="$(CommonPath)\System\Text\OSEncoding.Windows.cs">
<Link>Common\System\Text\OSEncoding.Windows.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\OSEncoder.cs">
<Link>Common\System\Text\OSEncoder.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\DBCSDecoder.cs">
<Link>Common\System\Text\DBCSDecoder.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.Encoding.Constants.cs">
<Link>Common\Interop\Windows\kernel32\Interop.Encoding.Constants.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.GetCPInfoEx.cs">
<Link>Common\Interop\Windows\kernel32\Interop.GetCPInfoEx.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\Interop.Libraries.cs">
<Link>Common\Interop\Windows\Interop.Libraries.cs</Link>
</Compile>
......@@ -106,6 +118,9 @@
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.HandleTypes.cs">
<Link>Common\Interop\Windows\Interop.HandleTypes.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.MultiByteToWideChar.cs">
<Link>Common\Interop\Windows\Interop.MultiByteToWideChar.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.PeekConsoleInput.cs">
<Link>Common\Interop\Windows\Interop.PeekConsoleInput.cs</Link>
</Compile>
......@@ -145,6 +160,9 @@
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.SetConsoleWindowInfo.cs">
<Link>Common\Interop\Windows\Interop.SetConsoleWindowInfo.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.WideCharToMultiByte.cs">
<Link>Common\Interop\Windows\Interop.WideCharToMultiByte.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.WriteFile_IntPtr.cs">
<Link>Common\Interop\Windows\Interop.WriteFile.cs</Link>
</Compile>
......@@ -154,6 +172,9 @@
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.WriteConsoleOutput.cs">
<Link>Common\Interop\Windows\Interop.WriteConsoleOutput.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\EncodingHelper.Windows.cs">
<Link>Common\System\IO\EncodingHelper.Windows.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\IO\Win32Marshal.cs">
<Link>Common\System\IO\Win32Marshal.cs</Link>
</Compile>
......
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
using Xunit;
public class ConsoleEncoding
{
public static IEnumerable<object[]> InputData()
{
yield return new object[] { "This is Ascii string" };
yield return new object[] { "This is string have surrogates \uD800\uDC00" };
yield return new object[] { "This string has non ascii charaters \u03b1\u00df\u0393\u03c0\u03a3\u03c3\u00b5" };
yield return new object[] { "This string has invalid surrogates \uD800\uD800\uD800\uD800\uD800\uD800" };
yield return new object[] { "\uD800" };
}
[Theory]
[PlatformSpecific(TestPlatforms.Windows)]
[MemberData(nameof(InputData))]
public void TestEncoding(string inputString)
{
var outConsoleStream = Console.Out;
var inConsoleStream = Console.In;
try
{
byte [] inputBytes = Console.OutputEncoding.GetBytes(inputString);
byte[] outBytes = new byte[inputBytes.Length];
using (MemoryStream ms = new MemoryStream(outBytes, true))
{
using (StreamWriter sw = new StreamWriter(ms, Console.OutputEncoding))
{
Console.SetOut(sw);
Console.Write(inputString);
}
}
Assert.Equal(inputBytes, outBytes);
string inString = new String(Console.InputEncoding.GetChars(inputBytes));
string outString;
using (MemoryStream ms = new MemoryStream(inputBytes, false))
{
using (StreamReader sr = new StreamReader(ms, Console.InputEncoding))
{
Console.SetIn(sr);
outString = Console.In.ReadToEnd();
}
}
Assert.Equal(inString, outString);
}
finally
{
Console.SetOut(outConsoleStream);
Console.SetIn(inConsoleStream);
}
}
}
......@@ -145,6 +145,14 @@ public static void CursorPositionAndArrowKeys()
AssertUserExpectedResults("the arrow keys move around the screen as expected with no other bad artificts");
}
[ConditionalFact(nameof(ManualTestsEnabled))]
public static void EncodingTest()
{
Console.WriteLine(Console.OutputEncoding);
Console.WriteLine("'\u03A0\u03A3'.");
AssertUserExpectedResults("Pi and Segma or question marks");
}
private static void AssertUserExpectedResults(string expected)
{
Console.Write($"Did you see {expected}? [y/n] ");
......
......@@ -27,6 +27,7 @@
<Compile Include="Color.cs" />
<Compile Include="SetOut.cs" />
<Compile Include="NegativeTesting.cs" />
<Compile Include="ConsoleEncoding.cs" />
<Compile Include="SyncTextReader.cs" />
<Compile Include="SyncTextWriter.cs" />
<Compile Include="Timeout.cs" />
......
......@@ -282,4 +282,34 @@
<data name="CantSetDuplicatePassword" xml:space="preserve">
<value>ProcessStartInfo.Password and ProcessStartInfo.PasswordInClearText cannot both be set. Use only one of them.</value>
</data>
<data name="ArgumentNull_Array" xml:space="preserve">
<value>Array cannot be null.</value>
</data>
<data name="ArgumentOutOfRange_IndexCountBuffer" xml:space="preserve">
<value>Index and count must refer to a location within the buffer.</value>
</data>
<data name="ArgumentOutOfRange_IndexCount" xml:space="preserve">
<value>Index and count must refer to a location within the string.</value>
</data>
<data name="ArgumentOutOfRange_Index" xml:space="preserve">
<value>Index was out of range. Must be non-negative and less than the size of the collection.</value>
</data>
<data name="Argument_EncodingConversionOverflowBytes" xml:space="preserve">
<value>The output byte buffer is too small to contain the encoded data, encoding '{0}' fallback '{1}'.</value>
</data>
<data name="Argument_EncodingConversionOverflowChars" xml:space="preserve">
<value>The output char buffer is too small to contain the decoded characters, encoding '{0}' fallback '{1}'.</value>
</data>
<data name="ArgumentOutOfRange_GetByteCountOverflow" xml:space="preserve">
<value>Too many characters. The resulting number of bytes is larger than what can be returned as an int.</value>
</data>
<data name="ArgumentOutOfRange_GetCharCountOverflow" xml:space="preserve">
<value>Too many bytes. The resulting number of chars is larger than what can be returned as an int.</value>
</data>
<data name="Argument_InvalidCharSequenceNoIndex" xml:space="preserve">
<value>String contains invalid Unicode code points.</value>
</data>
<data name="ArgumentOutOfRange_NeedNonNegNum" xml:space="preserve">
<value>Non-negative number required.</value>
</data>
</root>
\ No newline at end of file
......@@ -203,6 +203,9 @@
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.GetCurrentProcessId.cs">
<Link>Common\Interop\Windows\Interop.GetCurrentProcessId.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.Encoding.Constants.cs">
<Link>Common\Interop\Windows\kernel32\Interop.Encoding.Constants.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.GetConsoleCP.cs">
<Link>Common\Interop\Windows\Interop.GetConsoleCP.cs</Link>
</Compile>
......@@ -239,11 +242,29 @@
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.HandleOptions.cs">
<Link>Common\Interop\Windows\Interop.ProcessOptions.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.MultiByteToWideChar.cs">
<Link>Common\Interop\Windows\Interop.MultiByteToWideChar.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.WideCharToMultiByte.cs">
<Link>Common\Interop\Windows\Interop.WideCharToMultiByte.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\ConsoleEncoding.cs">
<Link>Common\System\Text\ConsoleEncoding.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\EncodingHelper.cs">
<Link>Common\System\Text\EncodingHelper.cs</Link>
<Compile Include="$(CommonPath)\System\Text\EncodingHelper.Windows.cs">
<Link>Common\System\Text\EncodingHelper.Windows.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\OSEncoding.Windows.cs">
<Link>Common\System\Text\OSEncoding.Windows.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\OSEncoder.cs">
<Link>Common\System\Text\OSEncoder.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\System\Text\DBCSDecoder.cs">
<Link>Common\System\Text\DBCSDecoder.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.GetCPInfoEx.cs">
<Link>Common\Interop\Windows\kernel32\Interop.GetCPInfoEx.cs</Link>
</Compile>
<Compile Include="Microsoft\Win32\SafeHandles\SafeProcessHandle.Windows.cs" />
<Compile Include="Microsoft\Win32\SafeHandles\SafeThreadHandle.cs" />
......
......@@ -43,22 +43,12 @@ public void TestChangesInConsoleEncoding()
try
{
{
Interop.SetConsoleCP(s_ConsoleEncoding);
Interop.SetConsoleOutputCP(s_ConsoleEncoding);
run(Encoding.UTF8.CodePage);
}
// Don't test this on Windows Nano, Windows Nano only supports UTF8.
if (File.Exists(Path.Combine(Environment.GetEnvironmentVariable("windir"), "regedit.exe")))
{
Interop.SetConsoleCP(s_ConsoleEncoding);
Interop.SetConsoleOutputCP(s_ConsoleEncoding);
// Register the codeprovider which will ensure 437 is enabled.
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
run(s_ConsoleEncoding);
}
}
......
......@@ -50,6 +50,9 @@
</ItemGroup>
<ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
<Compile Include="System\Text\CodePagesEncodingProvider.Windows.cs" />
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.Encoding.Constants.cs">
<Link>Common\Interop\Windows\kernel32\Interop.Encoding.Constants.cs</Link>
</Compile>
<Compile Include="$(CommonPath)\Interop\Windows\Interop.Libraries.cs">
<Link>Common\Interop\Windows\Interop.Libraries.cs</Link>
</Compile>
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册