Allow underscores in VB numeric literals

src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb

@@ -1554,6 +1554,7 @@ FullWidthRepeat:
 
             Dim Here As Integer = 0
             Dim IntegerLiteralStart As Integer
+            Dim UnderscoreInWrongPlace As Boolean
 
             Dim Base As LiteralBase = LiteralBase.Decimal
             Dim literalKind As NumericLiteralKind = NumericLiteralKind.Integral
@@ -1576,39 +1577,45 @@ FullWidthRepeat:
                         IntegerLiteralStart = Here
                         Base = LiteralBase.Hexadecimal
 
+                        UnderscoreInWrongPlace = Peek(Here) = "_"c
                         While CanGet(Here)
                             ch = Peek(Here)
-                            If Not IsHexDigit(ch) Then
+                            If Not IsHexDigit(ch) AndAlso ch <> "_"c Then
                                 Exit While
                             End If
                             Here += 1
                         End While
+                        UnderscoreInWrongPlace = UnderscoreInWrongPlace Or (Peek(Here - 1) = "_"c)
 
                     Case "B"c, "b"c
                         Here += 1
                         IntegerLiteralStart = Here
                         Base = LiteralBase.Binary
 
+                        UnderscoreInWrongPlace = Peek(Here) = "_"c
                         While CanGet(Here)
                             ch = Peek(Here)
-                            If Not IsBinaryDigit(ch) Then
+                            If Not IsBinaryDigit(ch) AndAlso ch <> "_"c Then
                                 Exit While
                             End If
                             Here += 1
                         End While
+                        UnderscoreInWrongPlace = UnderscoreInWrongPlace Or (Peek(Here - 1) = "_"c)
 
                     Case "O"c, "o"c
                         Here += 1
                         IntegerLiteralStart = Here
                         Base = LiteralBase.Octal
 
+                        UnderscoreInWrongPlace = Peek(Here) = "_"c
                         While CanGet(Here)
                             ch = Peek(Here)
-                            If Not IsOctalDigit(ch) Then
+                            If Not IsOctalDigit(ch) AndAlso ch <> "_"c Then
                                 Exit While
                             End If
                             Here += 1
                         End While
+                        UnderscoreInWrongPlace = UnderscoreInWrongPlace Or (Peek(Here - 1) = "_"c)
 
                     Case Else
                         If IsFullWidth(ch) Then
@@ -1621,13 +1628,17 @@ FullWidthRepeat:
             Else
                 ' no base specifier - just go through decimal digits.
                 IntegerLiteralStart = Here
+                UnderscoreInWrongPlace = Peek(Here) = "_"c
                 While CanGet(Here)
                     ch = Peek(Here)
-                    If Not IsDecimalDigit(ch) Then
+                    If Not IsDecimalDigit(ch) AndAlso ch <> "_"c Then
                         Exit While
                     End If
                     Here += 1
                 End While
+                If Here <> IntegerLiteralStart Then
+                    UnderscoreInWrongPlace = UnderscoreInWrongPlace Or (Peek(Here - 1) = "_"c)
+                End If
             End If
 
             ' we may have a dot, and then it is a float, but if this is an integral, then we have seen it all.
@@ -1647,11 +1658,12 @@ FullWidthRepeat:
                     ' all following decimal digits belong to the literal (fractional part)
                     While CanGet(Here)
                         ch = Peek(Here)
-                        If Not IsDecimalDigit(ch) Then
+                        If Not IsDecimalDigit(ch) AndAlso ch <> "_"c Then
                             Exit While
                         End If
                         Here += 1
                     End While
+                    UnderscoreInWrongPlace = UnderscoreInWrongPlace Or (Peek(Here - 1) = "_"c)
                     literalKind = NumericLiteralKind.Float
                 End If
 
@@ -1672,11 +1684,12 @@ FullWidthRepeat:
                         Here += 1
                         While CanGet(Here)
                             ch = Peek(Here)
-                            If Not IsDecimalDigit(ch) Then
+                            If Not IsDecimalDigit(ch) AndAlso ch <> "_"c Then
                                 Exit While
                             End If
                             Here += 1
                         End While
+                        UnderscoreInWrongPlace = UnderscoreInWrongPlace Or (Peek(Here - 1) = "_"c)
                     Else
                         Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_InvalidLiteralExponent)
                     End If
@@ -1819,12 +1832,16 @@ FullWidthRepeat2:
                 If IntegerLiteralStart = IntegerLiteralEnd Then
                     Return MakeBadToken(precedingTrivia, Here, ERRID.ERR_Syntax)
                 Else
-                    IntegralValue = IntegralLiteralCharacterValue(Peek(IntegerLiteralStart))
+                    IntegralValue = 0
 
                     If Base = LiteralBase.Decimal Then
                         ' Init For loop
-                        For LiteralCharacter As Integer = IntegerLiteralStart + 1 To IntegerLiteralEnd - 1
-                            Dim NextCharacterValue As UInteger = IntegralLiteralCharacterValue(Peek(LiteralCharacter))
+                        For LiteralCharacter As Integer = IntegerLiteralStart To IntegerLiteralEnd - 1
+                            Dim LiteralCharacterValue As Char = Peek(LiteralCharacter)
+                            If LiteralCharacterValue = "_"c Then
+                                Continue For
+                            End If
+                            Dim NextCharacterValue As UInteger = IntegralLiteralCharacterValue(LiteralCharacterValue)
 
                             If IntegralValue < 1844674407370955161UL OrElse
                               (IntegralValue = 1844674407370955161UL AndAlso NextCharacterValue <= 5UI) Then
@@ -1844,12 +1861,17 @@ FullWidthRepeat2:
                         Dim OverflowMask As UInt64 = If(Base = LiteralBase.Hexadecimal, &HF000000000000000UL, If(Base = LiteralBase.Octal, &HE000000000000000UL, &H8000000000000000UL))
 
                         ' Init For loop
-                        For LiteralCharacter As Integer = IntegerLiteralStart + 1 To IntegerLiteralEnd - 1
+                        For LiteralCharacter As Integer = IntegerLiteralStart To IntegerLiteralEnd - 1
+                            Dim LiteralCharacterValue As Char = Peek(LiteralCharacter)
+                            If LiteralCharacterValue = "_"c Then
+                                Continue For
+                            End If
+
                             If (IntegralValue And OverflowMask) <> 0 Then
                                 Overflows = True
                             End If
 
-                            IntegralValue = (IntegralValue << Shift) + IntegralLiteralCharacterValue(Peek(LiteralCharacter))
+                            IntegralValue = (IntegralValue << Shift) + IntegralLiteralCharacterValue(LiteralCharacterValue)
                         Next
                     End If
 
@@ -1892,7 +1914,9 @@ FullWidthRepeat2:
                 Dim scratch = GetScratch()
                 For i = 0 To literalWithoutTypeChar - 1
                     Dim curCh = Peek(i)
-                    scratch.Append(If(IsFullWidth(curCh), MakeHalfWidth(curCh), curCh))
+                    If curCh <> "_"c Then
+                        scratch.Append(If(IsFullWidth(curCh), MakeHalfWidth(curCh), curCh))
+                    End If
                 Next
                 Dim LiteralSpelling = GetScratchTextInterned(scratch)
 
@@ -1920,17 +1944,19 @@ FullWidthRepeat2:
             Dim result As SyntaxToken
             Select Case literalKind
                 Case NumericLiteralKind.Integral
-                    result = MakeIntegerLiteralToken(precedingTrivia, Base, TypeCharacter, If(Overflows, 0UL, IntegralValue), Here)
+                    result = MakeIntegerLiteralToken(precedingTrivia, Base, TypeCharacter, If(Overflows Or UnderscoreInWrongPlace, 0UL, IntegralValue), Here)
                 Case NumericLiteralKind.Float
-                    result = MakeFloatingLiteralToken(precedingTrivia, TypeCharacter, If(Overflows, 0.0F, FloatingValue), Here)
+                    result = MakeFloatingLiteralToken(precedingTrivia, TypeCharacter, If(Overflows Or UnderscoreInWrongPlace, 0.0F, FloatingValue), Here)
                 Case NumericLiteralKind.Decimal
-                    result = MakeDecimalLiteralToken(precedingTrivia, TypeCharacter, If(Overflows, 0D, DecimalValue), Here)
+                    result = MakeDecimalLiteralToken(precedingTrivia, TypeCharacter, If(Overflows Or UnderscoreInWrongPlace, 0D, DecimalValue), Here)
                 Case Else
                     Throw ExceptionUtilities.UnexpectedValue(literalKind)
             End Select
 
             If Overflows Then
                 result = DirectCast(result.AddError(ErrorFactory.ErrorInfo(ERRID.ERR_Overflow)), SyntaxToken)
+            ElseIf UnderscoreInWrongPlace Then
+                result = DirectCast(result.AddError(ErrorFactory.ErrorInfo(ERRID.ERR_Syntax)), SyntaxToken)
             End If
 
             Return result

src/Compilers/VisualBasic/Test/Syntax/Scanner/ScannerTests.vb

@@ -694,6 +694,13 @@ End If]]>.Value,
         Assert.Equal(42, tk.Value)
         Assert.Equal(" 42 ", tk.ToFullString())
 
+        Str = " 4_2 "
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
+        Assert.Equal(LiteralBase.Decimal, tk.GetBase())
+        Assert.Equal(42, tk.Value)
+        Assert.Equal(" 4_2 ", tk.ToFullString())
+
         Str = " &H42L "
         tk = ScanOnce(Str)
         Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
@@ -701,6 +708,13 @@ End If]]>.Value,
         Assert.Equal(&H42L, tk.Value)
         Assert.Equal(" &H42L ", tk.ToFullString())
 
+        Str = " &H4_2L "
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
+        Assert.Equal(LiteralBase.Hexadecimal, tk.GetBase())
+        Assert.Equal(&H42L, tk.Value)
+        Assert.Equal(" &H4_2L ", tk.ToFullString())
+
         Str = " &H42L &H42& "
         Dim tks = ScanAllCheckDw(Str)
         Assert.Equal(SyntaxKind.IntegerLiteralToken, tks(0).Kind)
@@ -714,6 +728,13 @@ End If]]>.Value,
         Assert.Equal(LiteralBase.Binary, tk.GetBase())
         Assert.Equal(&HAL, tk.Value)
         Assert.Equal(" &B1010L ", tk.ToFullString())
+
+        Str = " &B1_0_1_0L "
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
+        Assert.Equal(LiteralBase.Binary, tk.GetBase())
+        Assert.Equal(&HAL, tk.Value)
+        Assert.Equal(" &B1_0_1_0L ", tk.ToFullString())
     End Sub
 
     <Fact>
@@ -730,6 +751,13 @@ End If]]>.Value,
         Assert.IsType(Of Double)(tk.Value)
         Assert.Equal(" 0.42 ", tk.ToFullString())
 
+        Str = " 0_0.4_2 "
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.FloatingLiteralToken, tk.Kind)
+        Assert.Equal(0.42, tk.Value)
+        Assert.IsType(Of Double)(tk.Value)
+        Assert.Equal(" 0_0.4_2 ", tk.ToFullString())
+
         Str = " 0.42# "
         tk = ScanOnce(Str)
         Assert.Equal(SyntaxKind.FloatingLiteralToken, tk.Kind)
@@ -858,6 +886,44 @@ End If]]>.Value,
         Assert.Equal(0.0F, tk.Value)
     End Sub
 
+    <Fact>
+    Public Sub Scanner_UnderscoreWrongLocation()
+        Dim Str = "_1"
+        Dim tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.IdentifierToken, tk.Kind)
+        Assert.Equal(0, tk.GetSyntaxErrorsNoTree().Count())
+
+        Str = "1_"
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
+        Assert.Equal(30035, tk.GetSyntaxErrorsNoTree()(0).Code)
+        Assert.Equal(0, CInt(tk.Value))
+
+        Str = "&H_1"
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
+        Assert.Equal(30035, tk.GetSyntaxErrorsNoTree()(0).Code)
+        Assert.Equal(0, CInt(tk.Value))
+
+        Str = "&H1_"
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.IntegerLiteralToken, tk.Kind)
+        Assert.Equal(30035, tk.GetSyntaxErrorsNoTree()(0).Code)
+        Assert.Equal(0, CInt(tk.Value))
+
+        Str = "1_.1"
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.FloatingLiteralToken, tk.Kind)
+        Assert.Equal(30035, tk.GetSyntaxErrorsNoTree()(0).Code)
+        Assert.Equal(0, CInt(tk.Value))
+
+        Str = "1.1_"
+        tk = ScanOnce(Str)
+        Assert.Equal(SyntaxKind.FloatingLiteralToken, tk.Kind)
+        Assert.Equal(30035, tk.GetSyntaxErrorsNoTree()(0).Code)
+        Assert.Equal(0, CInt(tk.Value))
+    End Sub
+
     <Fact>
     Public Sub Scanner_DateLiteralToken()
         Dim Str = "#10/10/2010#"