Merge pull request #12954 from jaredpar/source

Split syntax generation into three files

build/Targets/GenerateCompilerInternals.props

+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
+  <!-- The syntax generation actually outputs three files: .Syntax, .Internal and .Main -->
+  <PropertyGroup Condition="'$(SyntaxDefinition)' != ''">
+      <SyntaxDefinitionFileName Condition="'$(SyntaxDefinition)' != ''">$([System.IO.Path]::GetFileName($(SyntaxDefinition)))</SyntaxDefinitionFileName>
+      <GeneratedSyntaxFileSyntax>$(IntermediateOutputPath)$(SyntaxDefinitionFileName).Syntax.Generated$(DefaultLanguageSourceExtension)</GeneratedSyntaxFileSyntax>
+      <GeneratedSyntaxFileMain>$(IntermediateOutputPath)$(SyntaxDefinitionFileName).Main.Generated$(DefaultLanguageSourceExtension)</GeneratedSyntaxFileMain>
+      <GeneratedSyntaxFileInternal>$(IntermediateOutputPath)$(SyntaxDefinitionFileName).Internal.Generated$(DefaultLanguageSourceExtension)</GeneratedSyntaxFileInternal>
+  </PropertyGroup>
+
+</Project>

build/Targets/GenerateCompilerInternals.targets

@@ -5,12 +5,6 @@
     <AvailableItemName Include="SyntaxTestDefinition" />
     <AvailableItemName Include="BoundTreeDefinition" />
   </ItemGroup>
-  <ItemDefinitionGroup>
-    <SyntaxDefinition>
-      <GenerateSource>True</GenerateSource>
-      <GenerateTests>False</GenerateTests>
-    </SyntaxDefinition>
-  </ItemDefinitionGroup>
 
   <PropertyGroup>
     <CompileDependsOn>
@@ -43,6 +37,12 @@
       <VBErrorFactsGeneratorToolPath>$(CompilerGeneratorToolsDir)VBErrorFactsGenerator.exe</VBErrorFactsGeneratorToolPath>
       <VBSyntaxGeneratorToolPath>$(CompilerGeneratorToolsDir)VBSyntaxGenerator.exe</VBSyntaxGeneratorToolPath>
       <BoundTreeGeneratorToolPath>$(CompilerGeneratorToolsDir)BoundTreeGenerator.exe</BoundTreeGeneratorToolPath>
+
+      <!-- The syntax generation actually outputs three files: .Syntax, .Internal and .Main -->
+      <SyntaxDefinitionFileName>$([System.IO.Path]::GetFileName($(SyntaxDefinition)))</SyntaxDefinitionFileName>
+      <GeneratedSyntaxFileSyntax>$(IntermediateOutputPath)$(SyntaxDefinitionFileName).Syntax.Generated$(DefaultLanguageSourceExtension)</GeneratedSyntaxFileSyntax>
+      <GeneratedSyntaxFileMain>$(IntermediateOutputPath)$(SyntaxDefinitionFileName).Main.Generated$(DefaultLanguageSourceExtension)</GeneratedSyntaxFileMain>
+      <GeneratedSyntaxFileInternal>$(IntermediateOutputPath)$(SyntaxDefinitionFileName).Internal.Generated$(DefaultLanguageSourceExtension)</GeneratedSyntaxFileInternal>
   </PropertyGroup>
 
   <Target Name="MarkCoreRunExecutable" Condition="'$(OS)' != 'Windows_NT' And Exists('$(CoreRunExe)')">
@@ -51,19 +51,18 @@
 
   <Target
     Name="GenerateSyntaxModel"
-    Inputs="@(SyntaxDefinition);$(VBSyntaxGeneratorToolPath);$(CSharpSyntaxGeneratorToolPath)"
-    Outputs="@(SyntaxDefinition -> '$(IntermediateOutputPath)%(Filename)%(Extension).Generated$(DefaultLanguageSourceExtension)')"
-    Condition="'@(SyntaxDefinition)' != '' AND '$(BuildingProject)' == 'true' AND ('$(Language)' == 'VB' or '$(Language)' == 'C#')"
+    Inputs="$(SyntaxDefinition);$(VBSyntaxGeneratorToolPath);$(CSharpSyntaxGeneratorToolPath)"
+    Outputs="$(GeneratedSyntaxFileSyntax);$(GeneratedSyntaxFileMain);$(GeneratedSyntaxFileInternal)"
+    Condition="'$(SyntaxDefinition)' != '' AND '$(BuildingProject)' == 'true' AND ('$(Language)' == 'VB' or '$(Language)' == 'C#')"
   >
     <PropertyGroup>
       <SyntaxGenerator Condition="'$(Language)' == 'VB'">"$(CoreRunExe)" "$(VBSyntaxGeneratorToolPath)"</SyntaxGenerator>
       <SyntaxGenerator Condition="'$(Language)' == 'C#'">"$(CoreRunExe)" "$(CSharpSyntaxGeneratorToolPath)"</SyntaxGenerator>
-      <GeneratedSyntaxModel>@(SyntaxDefinition -> '$(IntermediateOutputPath)%(Filename)%(Extension).Generated$(DefaultLanguageSourceExtension)')</GeneratedSyntaxModel>
     </PropertyGroup>
 
     <Exec
-      Command='$(SyntaxGenerator) "@(SyntaxDefinition)" "$(GeneratedSyntaxModel)"'
-      Outputs="$(GeneratedSyntaxModel)">
+      Command='$(SyntaxGenerator) "$(SyntaxDefinition)" "$(IntermediateOutputPath)"'
+      Outputs="$(GeneratedSyntaxFileSyntax);$(GeneratedSyntaxFileMain);$(GeneratedSyntaxFileInternal)">
       <Output TaskParameter="Outputs" ItemName="FileWrites" />
     </Exec>
 
@@ -72,11 +71,9 @@
   <Target
     Name="CleanSyntaxModel"
   >
-    <PropertyGroup>
-      <GeneratedSyntaxModel>@(SyntaxDefinition -> '$(IntermediateOutputPath)%(Filename)%(Extension).Generated$(DefaultLanguageSourceExtension)')</GeneratedSyntaxModel>
-    </PropertyGroup>
-
-    <Delete Files="$(GeneratedSyntaxModel)" />
+    <Delete Files="$(GeneratedSyntaxFileSyntax)" />
+    <Delete Files="$(GeneratedSyntaxFileMain)" />
+    <Delete Files="$(GeneratedSyntaxFileInternal)" />
   </Target>
 
   <Target

src/Compilers/CSharp/Portable/CSharpCodeAnalysis.csproj

@@ -19,6 +19,7 @@
     <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
     <RestorePackages>true</RestorePackages>
     <ServiceablePackage>true</ServiceablePackage>
+    <SyntaxDefinition>$(MSBuildThisFileDirectory)Syntax\Syntax.xml</SyntaxDefinition>
   </PropertyGroup>
   <ItemGroup Label="Project References">
     <ProjectReference Include="..\..\..\Tools\Source\CompilerGeneratorTools\DeployCompilerGeneratorToolsRuntime\DeployCompilerGeneratorToolsRuntime.csproj">
@@ -879,7 +880,9 @@
     </BoundTreeDefinition>
     <Compile Include="$(IntermediateOutputPath)BoundNodes.xml.Generated.cs" />
     <Compile Include="$(IntermediateOutputPath)ErrorFacts.Generated.cs" />
-    <Compile Include="$(IntermediateOutputPath)Syntax.xml.Generated.cs" />
+    <Compile Include="$(IntermediateOutputPath)Syntax.xml.Syntax.Generated.cs" />
+    <Compile Include="$(IntermediateOutputPath)Syntax.xml.Main.Generated.cs" />
+    <Compile Include="$(IntermediateOutputPath)Syntax.xml.Internal.Generated.cs" />
     <Compile Include="Errors\ErrorCode.cs" />
     <EmbeddedResource Include="CSharpResources.resx">
       <SubType>Designer</SubType>
@@ -887,9 +890,6 @@
       <LastGenOutput>CSharpResources.Designer.cs</LastGenOutput>
     </EmbeddedResource>
     <ErrorCode Include="Errors\ErrorCode.cs" />
-    <SyntaxDefinition Include="Syntax\Syntax.xml">
-      <SubType>Designer</SubType>
-    </SyntaxDefinition>
   </ItemGroup>
   <ItemGroup>
     <InternalsVisibleTo Include="Microsoft.CodeAnalysis.CSharp.ExpressionEvaluator.ExpressionCompiler" />
@@ -925,4 +925,4 @@
   <ImportGroup Label="Targets">
     <Import Project="..\..\..\..\build\Targets\VSL.Imports.targets" />
   </ImportGroup>
-</Project>
\ No newline at end of file
+</Project>

src/Compilers/VisualBasic/Portable/BasicCodeAnalysis.vbproj

@@ -17,6 +17,7 @@
     <ProjectTypeGuids>{14182A97-F7F0-4C62-8B27-98AA8AE2109A};{F184B08F-C81C-45F6-A57F-5ABD9991F28F}</ProjectTypeGuids>
     <RestorePackages>true</RestorePackages>
     <ServiceablePackage>true</ServiceablePackage>
+    <SyntaxDefinition>$(MSBuildThisFileDirectory)Syntax\Syntax.xml</SyntaxDefinition>
   </PropertyGroup>
   <ItemGroup Label="Project References">
     <ProjectReference Include="..\..\..\Tools\Source\CompilerGeneratorTools\DeployCompilerGeneratorToolsRuntime\DeployCompilerGeneratorToolsRuntime.csproj">
@@ -954,7 +955,9 @@
     <BoundTreeDefinition Include="BoundTree\BoundNodes.xml">
       <SubType>Designer</SubType>
     </BoundTreeDefinition>
-    <Compile Include="$(IntermediateOutputPath)Syntax.xml.Generated.vb" />
+    <Compile Include="$(IntermediateOutputPath)Syntax.xml.Syntax.Generated.vb" />
+    <Compile Include="$(IntermediateOutputPath)Syntax.xml.Main.Generated.vb" />
+    <Compile Include="$(IntermediateOutputPath)Syntax.xml.Internal.Generated.vb" />
     <Compile Include="$(IntermediateOutputPath)ErrorFacts.Generated.vb" />
     <Compile Include="Symbols\SymbolVisitor.vb" />
     <Compile Include="Symbols\SymbolVisitor`1.vb" />
@@ -965,9 +968,6 @@
     <PublicAPI Include="PublicAPI.Unshipped.txt" />
     <Content Include="Symbols\SymbolsAndNoPia.docx" />
     <ErrorCode Include="Errors\Errors.vb" />
-    <SyntaxDefinition Include="Syntax\Syntax.xml">
-      <SubType>Designer</SubType>
-    </SyntaxDefinition>
   </ItemGroup>
   <ItemGroup>
     <Import Include="IdentifierComparison = Microsoft.CodeAnalysis.CaseInsensitiveComparison" />
@@ -1016,4 +1016,4 @@
   <ImportGroup Label="Targets">
     <Import Project="..\..\..\..\build\Targets\VSL.Imports.targets" />
   </ImportGroup>
-</Project>
\ No newline at end of file
+</Project>

src/Tools/Source/CompilerGeneratorTools/Source/CSharpSyntaxGenerator/Program.cs

@@ -6,6 +6,7 @@
 using System.Linq;
 using System.Reflection;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Xml;
 using System.Xml.Serialization;
 
@@ -13,12 +14,12 @@ namespace CSharpSyntaxGenerator
 {
     internal static class Program
     {
-        public static void Main(string[] args)
+        public static int Main(string[] args)
         {
             if (args.Length < 2 || args.Length > 3)
             {
                 WriteUsage();
-                return;
+                return 1;
             }
 
             string inputFile = args[0];
@@ -26,7 +27,7 @@ public static void Main(string[] args)
             if (!File.Exists(inputFile))
             {
                 Console.WriteLine(inputFile + " not found.");
-                return;
+                return 1;
             }
 
             bool writeSource = true;
@@ -46,7 +47,7 @@ public static void Main(string[] args)
                 else
                 {
                     WriteUsage();
-                    return;
+                    return 1;
                 }
             }
             else if (args.Length == 2)
@@ -73,13 +74,23 @@ public static void Main(string[] args)
             {
                 if (writeSource)
                 {
-                    WriteToFile(tree, SourceWriter.Write, outputFile);
+                    var outputPath = outputFile.Trim('"');
+                    var prefix = Path.GetFileName(inputFile);
+                    var outputMainFile = Path.Combine(outputPath, $"{prefix}.Main.Generated.cs");
+                    var outputInternalFile = Path.Combine(outputPath, $"{prefix}.Internal.Generated.cs");
+                    var outputSyntaxFile = Path.Combine(outputPath, $"{prefix}.Syntax.Generated.cs");
+
+                    WriteToFile(tree, SourceWriter.WriteMain, outputMainFile);
+                    WriteToFile(tree, SourceWriter.WriteInternal, outputInternalFile);
+                    WriteToFile(tree, SourceWriter.WriteSyntax, outputSyntaxFile);
                 }
                 if (writeTests)
                 {
                     WriteToFile(tree, TestWriter.Write, outputFile);
                 }
             }
+
+            return 0;
         }
 
         private static void WriteUsage()

src/Tools/Source/CompilerGeneratorTools/Source/CSharpSyntaxGenerator/SourceWriter.cs

@@ -15,12 +15,13 @@ private SourceWriter(TextWriter writer, Tree tree)
         {
         }
 
-        public static void Write(TextWriter writer, Tree tree)
-        {
-            new SourceWriter(writer, tree).WriteFile();
-        }
+        public static void WriteMain(TextWriter writer, Tree tree) => new SourceWriter(writer, tree).WriteMain();
+
+        public static void WriteInternal(TextWriter writer, Tree tree) => new SourceWriter(writer, tree).WriteInternal();
+
+        public static void WriteSyntax(TextWriter writer, Tree tree) => new SourceWriter(writer, tree).WriteSyntax();
 
-        private void WriteFile()
+        private void WriteFileHeader()
         {
             WriteLine("// <auto-generated />");
             WriteLine();
@@ -31,6 +32,11 @@ private void WriteFile()
             WriteLine("using System.Threading;");
             WriteLine("using Roslyn.Utilities;");
             WriteLine();
+        }
+
+        private void WriteInternal()
+        {
+            WriteFileHeader();
 
             WriteLine("namespace Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax");
             WriteLine("{");
@@ -41,6 +47,11 @@ private void WriteFile()
             this.WriteContextualGreenFactories();
             this.WriteStaticGreenFactories();
             WriteLine("}");
+        }
+
+        private void WriteSyntax()
+        {
+            WriteFileHeader();
 
             WriteLine();
             WriteLine("namespace Microsoft.CodeAnalysis.CSharp.Syntax");
@@ -48,6 +59,11 @@ private void WriteFile()
             WriteLine();
             this.WriteRedTypes();
             WriteLine("}");
+        }
+
+        private void WriteMain()
+        {
+            WriteFileHeader();
 
             WriteLine();
             WriteLine("namespace Microsoft.CodeAnalysis.CSharp");

src/Tools/Source/CompilerGeneratorTools/Source/VisualBasicSyntaxGenerator/Program.vb

@@ -56,7 +56,7 @@ Friend Module Program
             End If
 
             Dim checksum = GetChecksum(inputFile)
-            WriteOutput(outputFile, definition, outputKind, checksum)
+            WriteOutput(inputFile, outputFile, definition, outputKind, checksum)
 
             Return exitWithoutErrors
 
@@ -94,16 +94,12 @@ Friend Module Program
         Return True
     End Function
 
-    Public Sub WriteOutput(outputFile As String, definition As ParseTree, outputKind As String, checksum As String)
+    Public Sub WriteOutput(inputFile As String, outputFile As String, definition As ParseTree, outputKind As String, checksum As String)
+        Select Case outputKind
+            Case "/test"
+                Using output As New StreamWriter(New FileStream(outputFile, FileMode.Create, FileAccess.Write))
 
-        Using output As New StreamWriter(New FileStream(outputFile, FileMode.Create, FileAccess.Write))
-            output.WriteLine("' Definition of syntax model.")
-            output.WriteLine("' Generated by a tool from SHA256 content {0}", checksum)
-            output.WriteLine("' DO NOT HAND EDIT")
-
-
-            Select Case outputKind
-                Case "/test"
+                    WriteHeader(output, checksum)
                     output.WriteLine()
                     output.WriteLine("Imports System.Collections.Generic")
                     output.WriteLine("Imports System.Collections.Immutable")
@@ -114,38 +110,73 @@ Friend Module Program
 
                     Dim testWriter As New TestWriter(definition, checksum)
                     testWriter.WriteTestCode(output)
+                End Using
 
-                Case "/gettext"
+            Case "/gettext"
+                Using output As New StreamWriter(New FileStream(outputFile, FileMode.Create, FileAccess.Write))
+                    WriteHeader(output, checksum)
                     Dim syntaxFactsWriter As New SyntaxFactsWriter(definition)
                     syntaxFactsWriter.GenerateGetText(output)
+                End Using
 
-                Case Else
-                    output.WriteLine()
-                    output.WriteLine("Imports System.Collections.Generic")
-                    output.WriteLine("Imports System.Collections.Immutable")
-                    output.WriteLine("Imports System.Runtime.CompilerServices")
-                    output.WriteLine("Imports Microsoft.CodeAnalysis.VisualBasic.Syntax")
-                    output.WriteLine("Imports Roslyn.Utilities")
+            Case Else
+                WriteSyntax(inputFile, outputFile, definition, checksum)
+        End Select
+    End Sub
+
+    Public Sub WriteSyntax(inputFile As String, outputFile As String, definition As ParseTree, checksum As String)
+        Dim outputPath = outputFile.Trim(""""c)
+        Dim prefix = Path.GetFileName(inputFile)
+        Dim mainFile = Path.Combine(outputPath, $"{prefix}.Main.Generated.vb")
+        Dim syntaxFile = Path.Combine(outputPath, $"{prefix}.Syntax.Generated.vb")
+        Dim internalFile = Path.Combine(outputPath, $"{prefix}.Internal.Generated.vb")
+        Dim redNodeWriter As New RedNodeWriter(definition)
 
-                    Dim redNodeWriter As New RedNodeWriter(definition)
-                    redNodeWriter.WriteTreeAsCode(output)
+        Using output As New StreamWriter(New FileStream(mainFile, FileMode.Create, FileAccess.Write))
+            WriteSyntaxHeader(output, checksum)
+            redNodeWriter.WriteMainTreeAsCode(output)
 
-                    Dim greenNodeWriter As New GreenNodeWriter(definition)
-                    greenNodeWriter.WriteTreeAsCode(output)
+            Dim redFactoryWriter As New RedNodeFactoryWriter(definition)
+            redFactoryWriter.WriteFactories(output)
 
-                    Dim redFactoryWriter As New RedNodeFactoryWriter(definition)
-                    redFactoryWriter.WriteFactories(output)
+            Dim syntaxFactsWriter As New SyntaxFactsWriter(definition)
+            syntaxFactsWriter.GenerateFile(output)
+        End Using
 
-                    Dim greenFactoryWriter As New GreenNodeFactoryWriter(definition)
-                    greenFactoryWriter.WriteFactories(output)
+        Using output As New StreamWriter(New FileStream(syntaxFile, FileMode.Create, FileAccess.Write))
+            WriteSyntaxHeader(output, checksum)
 
-                    Dim syntaxFactsWriter As New SyntaxFactsWriter(definition)
-                    syntaxFactsWriter.GenerateFile(output)
+            redNodeWriter.WriteSyntaxTreeAsCode(output)
+        End Using
 
-            End Select
+        Using output As New StreamWriter(New FileStream(internalFile, FileMode.Create, FileAccess.Write))
+            WriteSyntaxHeader(output, checksum)
 
+            Dim greenNodeWriter As New GreenNodeWriter(definition)
+            greenNodeWriter.WriteTreeAsCode(output)
+
+            Dim greenFactoryWriter As New GreenNodeFactoryWriter(definition)
+            greenFactoryWriter.WriteFactories(output)
         End Using
 
     End Sub
 
+    Private Sub WriteHeader(output As StreamWriter, checksum As String)
+        output.WriteLine("' Definition of syntax model.")
+        output.WriteLine("' Generated by a tool from SHA256 content {0}", checksum)
+        output.WriteLine("' DO NOT HAND EDIT")
+    End Sub
+
+
+    Private Sub WriteSyntaxHeader(output As StreamWriter, checksum As String)
+        WriteHeader(output, checksum)
+
+        output.WriteLine()
+        output.WriteLine("Imports System.Collections.Generic")
+        output.WriteLine("Imports System.Collections.Immutable")
+        output.WriteLine("Imports System.Runtime.CompilerServices")
+        output.WriteLine("Imports Microsoft.CodeAnalysis.VisualBasic.Syntax")
+        output.WriteLine("Imports Roslyn.Utilities")
+    End Sub
+
 End Module

src/Tools/Source/CompilerGeneratorTools/Source/VisualBasicSyntaxGenerator/RedNodes/RedNodeWriter.vb

@@ -14,18 +14,17 @@ Friend Class RedNodeWriter
     End Sub
 
     ' Write out the code defining the tree to the give file.
-    Public Sub WriteTreeAsCode(writer As TextWriter)
+    Public Sub WriteMainTreeAsCode(writer As TextWriter)
         _writer = writer
-
-        GenerateFile()
+        GenerateMainNamespace()
     End Sub
 
-    Private Sub GenerateFile()
-        GenerateNamespace()
+    Public Sub WriteSyntaxTreeAsCode(writer As TextWriter)
+        _writer = writer
+        GenerateSyntaxNamespace()
     End Sub
 
-    Private Sub GenerateNamespace()
-
+    Private Sub GenerateMainNamespace()
         If Not String.IsNullOrEmpty(_parseTree.NamespaceName) Then
             _writer.WriteLine()
             _writer.WriteLine("Namespace {0}", Ident(_parseTree.NamespaceName))
@@ -49,9 +48,11 @@ Friend Class RedNodeWriter
         If Not String.IsNullOrEmpty(_parseTree.NamespaceName) Then
             _writer.WriteLine("End Namespace")
         End If
+    End Sub
 
-        _writer.WriteLine()
+    Private Sub GenerateSyntaxNamespace()
         If Not String.IsNullOrEmpty(_parseTree.NamespaceName) Then
+            _writer.WriteLine()
             _writer.WriteLine("Namespace {0}", Ident(_parseTree.NamespaceName & ".Syntax"))
             _writer.WriteLine()
         End If