提交 7dd7af9f 编写于 作者: J Julien Couvreur 提交者: GitHub

Merge pull request #20449 from alrz/features/leading-digit-separator

Allow digit separator after base specifier
Merging on behalf of @alrz. Thanks for the contribution!
......@@ -10223,6 +10223,15 @@ internal class CSharpResources {
}
}
/// <summary>
/// Looks up a localized string similar to leading digit separator.
/// </summary>
internal static string IDS_FeatureLeadingDigitSeparator {
get {
return ResourceManager.GetString("IDS_FeatureLeadingDigitSeparator", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to local functions.
/// </summary>
......
......@@ -5111,4 +5111,7 @@ To remove the warning, you can use /reference instead (set the Embed Interop Typ
<data name="ERR_DynamicLocalFunctionTypeParameter" xml:space="preserve">
<value>Cannot pass argument with dynamic type to generic local function '{0}' with inferred type arguments.</value>
</data>
<data name="IDS_FeatureLeadingDigitSeparator" xml:space="preserve">
<value>leading digit separator</value>
</data>
</root>
\ No newline at end of file
......@@ -132,6 +132,8 @@ internal enum MessageID
IDS_FeatureGenericPatternMatching = MessageBase + 12720,
IDS_FeatureAsyncMain = MessageBase + 12721,
IDS_LangVersions = MessageBase + 12722,
IDS_FeatureLeadingDigitSeparator = MessageBase + 12723,
}
// Message IDs may refer to strings that need to be localized.
......@@ -188,6 +190,10 @@ internal static LanguageVersion RequiredVersion(this MessageID feature)
// Checks are in the LanguageParser unless otherwise noted.
switch (feature)
{
// C# 7.2 features.
case MessageID.IDS_FeatureLeadingDigitSeparator:
return LanguageVersion.CSharp7_2;
// C# 7.1 features.
case MessageID.IDS_FeatureAsyncMain:
case MessageID.IDS_FeatureDefaultLiteral:
......
......@@ -932,27 +932,34 @@ private bool ScanInteger()
return start < TextWindow.Position;
}
// Allows underscores in integers, except at beginning and end
// Allows underscores in integers, except at beginning for decimal and end
private void ScanNumericLiteralSingleInteger(ref bool underscoreInWrongPlace, ref bool usedUnderscore, bool isHex, bool isBinary)
{
bool firstCharWasUnderscore = false;
if (TextWindow.PeekChar() == '_')
{
underscoreInWrongPlace = true;
if (isHex || isBinary)
{
firstCharWasUnderscore = true;
}
else
{
underscoreInWrongPlace = true;
}
}
char ch;
var lastCharWasUnderscore = false;
bool lastCharWasUnderscore = false;
while (true)
{
ch = TextWindow.PeekChar();
char ch = TextWindow.PeekChar();
if (ch == '_')
{
usedUnderscore = true;
lastCharWasUnderscore = true;
}
else if ((isHex && !SyntaxFacts.IsHexDigit(ch))
|| (isBinary && !SyntaxFacts.IsBinaryDigit(ch))
|| (!isHex && !isBinary && !SyntaxFacts.IsDecDigit(ch)))
else if (!(isHex ? SyntaxFacts.IsHexDigit(ch) :
isBinary ? SyntaxFacts.IsBinaryDigit(ch) :
SyntaxFacts.IsDecDigit(ch)))
{
break;
}
......@@ -964,7 +971,13 @@ private void ScanNumericLiteralSingleInteger(ref bool underscoreInWrongPlace, re
TextWindow.AdvanceChar();
}
if (lastCharWasUnderscore)
if (firstCharWasUnderscore)
{
CheckFeatureAvailability(MessageID.IDS_FeatureLeadingDigitSeparator);
// No need for cascading feature error
usedUnderscore = false;
}
else if (lastCharWasUnderscore)
{
underscoreInWrongPlace = true;
}
......
......@@ -16,6 +16,8 @@ public class LexicalTests
{
private readonly CSharpParseOptions _options;
private readonly CSharpParseOptions _options6;
private readonly CSharpParseOptions _options7;
private readonly CSharpParseOptions _options72;
private readonly CSharpParseOptions _binaryOptions;
private readonly CSharpParseOptions _underscoreOptions;
private readonly CSharpParseOptions _binaryUnderscoreOptions;
......@@ -24,8 +26,10 @@ public LexicalTests()
{
_options = new CSharpParseOptions(languageVersion: LanguageVersion.CSharp3);
_options6 = new CSharpParseOptions(languageVersion: LanguageVersion.CSharp6);
_binaryOptions = _options.WithLanguageVersion(LanguageVersion.CSharp7);
_underscoreOptions = _options.WithLanguageVersion(LanguageVersion.CSharp7);
_options7 = new CSharpParseOptions(languageVersion: LanguageVersion.CSharp7);
_options72 = new CSharpParseOptions(languageVersion: LanguageVersion.CSharp7_2);
_binaryOptions = _options7;
_underscoreOptions = _options7;
_binaryUnderscoreOptions = _binaryOptions;
}
......@@ -2614,6 +2618,89 @@ public void TestNumericWithUnderscoresWithoutFeatureFlag()
Assert.Equal(text, token.Text);
}
[Fact]
[Trait("Feature", "Literals")]
public void TestNumericWithLeadingUnderscores()
{
var text = "0x_A";
var token = LexToken(text, _options72);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
Assert.Equal(0, token.Errors().Length);
Assert.Equal(0xA, token.Value);
Assert.Equal(text, token.Text);
text = "0b_1";
token = LexToken(text, _options72);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
Assert.Equal(0, token.Errors().Length);
Assert.Equal(1, token.Value);
Assert.Equal(text, token.Text);
text = "0x__A_1L";
token = LexToken(text, _options72);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
Assert.Equal(0, token.Errors().Length);
Assert.Equal(0xA1L, token.Value);
Assert.Equal(text, token.Text);
text = "0b__1_1L";
token = LexToken(text, _options72);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
Assert.Equal(0, token.Errors().Length);
Assert.Equal(0b11L, token.Value);
Assert.Equal(text, token.Text);
}
[Fact]
[Trait("Feature", "Literals")]
public void TestNumericWithLeadingUnderscoresWithoutFeatureFlag()
{
var text = "0x_A";
var token = LexToken(text, _options7);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
var errors = token.Errors();
Assert.Equal(1, errors.Length);
Assert.Equal((int)ErrorCode.ERR_FeatureNotAvailableInVersion7, errors[0].Code);
Assert.Equal(text, token.Text);
text = "0b_1";
token = LexToken(text, _options7);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
errors = token.Errors();
Assert.Equal(1, errors.Length);
Assert.Equal((int)ErrorCode.ERR_FeatureNotAvailableInVersion7, errors[0].Code);
Assert.Equal(text, token.Text);
text = "0x_1";
token = LexToken(text, _options6);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
errors = token.Errors();
Assert.Equal(1, errors.Length);
Assert.Equal((int)ErrorCode.ERR_FeatureNotAvailableInVersion6, errors[0].Code);
Assert.Equal(text, token.Text);
text = "0x_123_456_789_ABC_DEF_123";
token = LexToken(text, _options6);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
errors = token.Errors();
Assert.Equal(2, errors.Length);
Assert.Equal((int)ErrorCode.ERR_FeatureNotAvailableInVersion6, errors[0].Code);
Assert.Equal((int)ErrorCode.ERR_IntOverflow, errors[1].Code);
Assert.Equal(text, token.Text);
}
[Fact]
[Trait("Feature", "Literals")]
public void TestNumericWithBadUnderscores()
......@@ -2695,7 +2782,7 @@ public void TestNumericWithBadUnderscores()
Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
Assert.Equal(text, token.Text);
text = "0x_A";
text = "0xA_";
token = LexToken(text, _underscoreOptions);
Assert.NotNull(token);
......@@ -2706,8 +2793,8 @@ public void TestNumericWithBadUnderscores()
Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
Assert.Equal(text, token.Text);
text = "0xA_";
token = LexToken(text, _underscoreOptions);
text = "0b1_";
token = LexToken(text, _binaryUnderscoreOptions);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
......@@ -2717,8 +2804,8 @@ public void TestNumericWithBadUnderscores()
Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
Assert.Equal(text, token.Text);
text = "0b_1";
token = LexToken(text, _binaryUnderscoreOptions);
text = "0x_";
token = LexToken(text, _options72);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
......@@ -2728,8 +2815,8 @@ public void TestNumericWithBadUnderscores()
Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
Assert.Equal(text, token.Text);
text = "0b1_";
token = LexToken(text, _binaryUnderscoreOptions);
text = "1E+_2";
token = LexToken(text, _options72);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
......@@ -2738,6 +2825,29 @@ public void TestNumericWithBadUnderscores()
Assert.Equal((int)ErrorCode.ERR_InvalidNumber, errors[0].Code);
Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
Assert.Equal(text, token.Text);
text = "1E-_2";
token = LexToken(text, _options72);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
errors = token.Errors();
Assert.Equal(1, errors.Length);
Assert.Equal((int)ErrorCode.ERR_InvalidNumber, errors[0].Code);
Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
Assert.Equal(text, token.Text);
text = "1E_";
token = LexToken(text, _options72);
Assert.NotNull(token);
Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
errors = token.Errors();
Assert.Equal(2, errors.Length);
Assert.Equal((int)ErrorCode.ERR_InvalidNumber, errors[0].Code);
Assert.Equal((int)ErrorCode.ERR_FloatOverflow, errors[1].Code);
Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
Assert.Equal(text, token.Text);
}
[Fact]
......
......@@ -2009,5 +2009,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
FEATURE_BinaryLiterals
FEATURE_Tuples
FEATURE_IOperation
FEATURE_LeadingDigitSeparator
End Enum
End Namespace
......@@ -34,6 +34,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Tuples
IOperation
InferredTupleNames
LeadingDigitSeparator
End Enum
Friend Module FeatureExtensions
......@@ -90,6 +91,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Case Feature.InferredTupleNames
Return LanguageVersion.VisualBasic15_3
Case Feature.LeadingDigitSeparator
Return LanguageVersion.VisualBasic15_5
Case Else
Throw ExceptionUtilities.UnexpectedValue(feature)
End Select
......@@ -153,6 +157,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
Return ERRID.FEATURE_Tuples
Case Feature.IOperation
Return ERRID.FEATURE_IOperation
Case Feature.LeadingDigitSeparator
Return ERRID.FEATURE_LeadingDigitSeparator
Case Else
Throw ExceptionUtilities.UnexpectedValue(feature)
End Select
......
......@@ -1673,6 +1673,7 @@ FullWidthRepeat:
Dim IntegerLiteralStart As Integer
Dim UnderscoreInWrongPlace As Boolean
Dim UnderscoreUsed As Boolean = False
Dim LeadingUnderscoreUsed = False
Dim Base As LiteralBase = LiteralBase.Decimal
Dim literalKind As NumericLiteralKind = NumericLiteralKind.Integral
......@@ -1695,7 +1696,10 @@ FullWidthRepeat:
IntegerLiteralStart = Here
Base = LiteralBase.Hexadecimal
UnderscoreInWrongPlace = (CanGet(Here) AndAlso Peek(Here) = "_"c)
If CanGet(Here) AndAlso Peek(Here) = "_"c Then
LeadingUnderscoreUsed = True
End If
While CanGet(Here)
ch = Peek(Here)
If Not IsHexDigit(ch) AndAlso ch <> "_"c Then
......@@ -1713,7 +1717,10 @@ FullWidthRepeat:
IntegerLiteralStart = Here
Base = LiteralBase.Binary
UnderscoreInWrongPlace = (CanGet(Here) AndAlso Peek(Here) = "_"c)
If CanGet(Here) AndAlso Peek(Here) = "_"c Then
LeadingUnderscoreUsed = True
End If
While CanGet(Here)
ch = Peek(Here)
If Not IsBinaryDigit(ch) AndAlso ch <> "_"c Then
......@@ -1731,7 +1738,10 @@ FullWidthRepeat:
IntegerLiteralStart = Here
Base = LiteralBase.Octal
UnderscoreInWrongPlace = (CanGet(Here) AndAlso Peek(Here) = "_"c)
If CanGet(Here) AndAlso Peek(Here) = "_"c Then
LeadingUnderscoreUsed = True
End If
While CanGet(Here)
ch = Peek(Here)
If Not IsOctalDigit(ch) AndAlso ch <> "_"c Then
......@@ -2085,13 +2095,16 @@ FullWidthRepeat2:
If Overflows Then
result = DirectCast(result.AddError(ErrorFactory.ErrorInfo(ERRID.ERR_Overflow)), SyntaxToken)
ElseIf UnderscoreInWrongPlace Then
result = DirectCast(result.AddError(ErrorFactory.ErrorInfo(ERRID.ERR_Syntax)), SyntaxToken)
End If
If UnderscoreUsed Then
If UnderscoreInWrongPlace Then
result = DirectCast(result.AddError(ErrorFactory.ErrorInfo(ERRID.ERR_Syntax)), SyntaxToken)
ElseIf LeadingUnderscoreUsed Then
result = CheckFeatureAvailability(result, Feature.LeadingDigitSeparator)
ElseIf UnderscoreUsed Then
result = CheckFeatureAvailability(result, Feature.DigitSeparators)
End If
If Base = LiteralBase.Binary Then
result = CheckFeatureAvailability(result, Feature.BinaryLiterals)
End If
......
......@@ -12036,6 +12036,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
End Get
End Property
'''<summary>
''' Looks up a localized string similar to leading digit separator.
'''</summary>
Friend ReadOnly Property FEATURE_LeadingDigitSeparator() As String
Get
Return ResourceManager.GetString("FEATURE_LeadingDigitSeparator", resourceCulture)
End Get
End Property
'''<summary>
''' Looks up a localized string similar to implicit line continuation.
'''</summary>
......
......@@ -5500,5 +5500,8 @@
<data name="ERR_InvalidDebugInfo" xml:space="preserve">
<value>Unable to read debug information of method '{0}' (token 0x{1}) from assembly '{2}'</value>
</data>
<data name="FEATURE_LeadingDigitSeparator" xml:space="preserve">
<value>leading digit separator</value>
</data>
</root>
......@@ -16,6 +16,10 @@ Public Class ScannerTests
Return SyntaxFactory.ParseToken(str, startStatement:=startStatement)
End Function
Private Function ScanOnce(str As String, languageVersion As VisualBasic.LanguageVersion) As SyntaxToken
Return SyntaxFactory.ParseTokens(str, options:=New VisualBasicParseOptions(languageVersion:=languageVersion)).First()
End Function
Private Function AsString(tokens As IEnumerable(Of SyntaxToken)) As String
Dim str = String.Concat(From t In tokens Select t.ToFullString())
Return str
......@@ -1042,6 +1046,48 @@ End If]]>.Value,
Assert.Equal(&H42L, tk.Value)
Assert.Equal(" &H4_2L ", tk.ToFullString())
Str = " &H_1 "
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Assert.Equal(LiteralBase.Hexadecimal, tk.GetBase())
Assert.Equal(&H1, tk.Value)
Assert.Equal(" &H_1 ", tk.ToFullString())
Str = " &B_1 "
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Assert.Equal(LiteralBase.Binary, tk.GetBase())
Assert.Equal(&B1, tk.Value)
Assert.Equal(" &B_1 ", tk.ToFullString())
Str = " &O_1 "
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Assert.Equal(LiteralBase.Octal, tk.GetBase())
Assert.Equal(&O1, tk.Value)
Assert.Equal(" &O_1 ", tk.ToFullString())
Str = " &H__1_1L "
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Assert.Equal(LiteralBase.Hexadecimal, tk.GetBase())
Assert.Equal(&H11L, tk.Value)
Assert.Equal(" &H__1_1L ", tk.ToFullString())
Str = " &B__1_1L "
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Assert.Equal(LiteralBase.Binary, tk.GetBase())
Assert.Equal(&B11L, tk.Value)
Assert.Equal(" &B__1_1L ", tk.ToFullString())
Str = " &O__1_1L "
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Assert.Equal(LiteralBase.Octal, tk.GetBase())
Assert.Equal(&O11L, tk.Value)
Assert.Equal(" &O__1_1L ", tk.ToFullString())
Str = " &H42L &H42& "
Dim tks = ScanAllCheckDw(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tks(0).Kind)
......@@ -1227,12 +1273,6 @@ End If]]>.Value,
Assert.Equal(30035, tk.GetSyntaxErrorsNoTree()(0).Code)
Assert.Equal(0, CInt(tk.Value))
Str = "&H_1"
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Assert.Equal(30035, tk.GetSyntaxErrorsNoTree()(0).Code)
Assert.Equal(0, CInt(tk.Value))
Str = "&H1_"
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
......@@ -1250,6 +1290,34 @@ End If]]>.Value,
Assert.Equal(SyntaxKind.FloatingLiteralToken, tk.Kind)
Assert.Equal(30035, tk.GetSyntaxErrorsNoTree()(0).Code)
Assert.Equal(0, CInt(tk.Value))
Str = "&H_"
tk = ScanOnce(Str)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Dim errors = tk.Errors()
Assert.Equal(1, errors.Count)
Assert.Equal(30035, errors.First().Code)
Assert.Equal(0, CInt(tk.Value))
End Sub
<Fact>
Public Sub Scanner_UnderscoreFeatureFlag()
Dim Str = "&H_1"
Dim tk = ScanOnce(Str, LanguageVersion.VisualBasic14)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
Dim errors = tk.Errors()
Assert.Equal(1, errors.Count)
Assert.Equal(36716, errors.First().Code)
Assert.Equal(1, CInt(tk.Value))
Str = "&H_123_456_789_ABC_DEF_123"
tk = ScanOnce(Str, LanguageVersion.VisualBasic14)
Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
errors = tk.Errors()
Assert.Equal(2, errors.Count)
Assert.Equal(30036, errors.ElementAt(0).Code)
Assert.Equal(36716, errors.ElementAt(1).Code)
Assert.Equal(0, CInt(tk.Value))
End Sub
<Fact>
......@@ -1751,4 +1819,17 @@ Module SyntaxDiagnosticInfoListExtensions
Throw New InvalidOperationException()
End Function
<Extension>
Public Function ElementAt(list As SyntaxDiagnosticInfoList, index As Integer) As DiagnosticInfo
Dim i = 0
For Each v In list
If i = index Then
Return v
End If
i += 1
Next
Throw New IndexOutOfRangeException()
End Function
End Module
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册