diff --git a/Compilers.sln b/Compilers.sln index 838fac521c23693416209b2f0d0300232ab58317..3488eff78a4524f8d13fde06b3ff57c4a0867c23 100644 --- a/Compilers.sln +++ b/Compilers.sln @@ -146,6 +146,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IlAsmDeploy", "src\Tools\IL EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NuGetProjectPackUtil", "src\NuGet\NuGetProjectPackUtil.csproj", "{B2B261E8-56EC-45DC-8AB8-A491A065EFD4}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CompilerBenchmarks", "src\Tools\CompilerBenchmarks\CompilerBenchmarks.csproj", "{B446E771-AB52-41C9-ACFC-FDF8EACAF291}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\Compilers\Core\AnalyzerDriver\AnalyzerDriver.projitems*{1ee8cad3-55f9-4d91-96b2-084641da9a6c}*SharedItemsImports = 4 @@ -397,6 +399,10 @@ Global {B2B261E8-56EC-45DC-8AB8-A491A065EFD4}.Debug|Any CPU.Build.0 = Debug|Any CPU {B2B261E8-56EC-45DC-8AB8-A491A065EFD4}.Release|Any CPU.ActiveCfg = Release|Any CPU {B2B261E8-56EC-45DC-8AB8-A491A065EFD4}.Release|Any CPU.Build.0 = Release|Any CPU + {B446E771-AB52-41C9-ACFC-FDF8EACAF291}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B446E771-AB52-41C9-ACFC-FDF8EACAF291}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B446E771-AB52-41C9-ACFC-FDF8EACAF291}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B446E771-AB52-41C9-ACFC-FDF8EACAF291}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -464,6 +470,7 @@ Global {E8F0BAA5-7327-43D1-9A51-644E81AE55F1} = {C65C6143-BED3-46E6-869E-9F0BE6E84C37} {46B3E63A-C462-4133-9F27-3B85DA5E7D37} = {FD0FAF5F-1DED-485C-99FA-84B97F3A8EEC} {B2B261E8-56EC-45DC-8AB8-A491A065EFD4} = {FD0FAF5F-1DED-485C-99FA-84B97F3A8EEC} + {B446E771-AB52-41C9-ACFC-FDF8EACAF291} = {FD0FAF5F-1DED-485C-99FA-84B97F3A8EEC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {6F599E08-A9EA-4FAA-897F-5D824B0210E6} diff --git a/Roslyn.sln b/Roslyn.sln index 55411732346cb461feaa2851b4879bd7fcabaf4d..cd9ba38837780ed8fbdde4e4e00852ffee9e3740 100644 --- a/Roslyn.sln +++ b/Roslyn.sln @@ -371,6 +371,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "InteractiveHost64", "src\In EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.VisualStudio.IntegrationTest.IntegrationService", "src\VisualStudio\IntegrationTest\IntegrationService\Microsoft.VisualStudio.IntegrationTest.IntegrationService.csproj", "{764D2C19-0187-4837-A2A3-96DDC6EF4CE2}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CompilerBenchmarks", "src\Tools\CompilerBenchmarks\CompilerBenchmarks.csproj", "{9860FCF7-3111-4C12-A16F-ACEBA42D930F}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\Compilers\VisualBasic\BasicAnalyzerDriver\BasicAnalyzerDriver.projitems*{2523d0e6-df32-4a3e-8ae0-a19bffae2ef6}*SharedItemsImports = 4 @@ -992,6 +994,10 @@ Global {764D2C19-0187-4837-A2A3-96DDC6EF4CE2}.Debug|Any CPU.Build.0 = Debug|Any CPU {764D2C19-0187-4837-A2A3-96DDC6EF4CE2}.Release|Any CPU.ActiveCfg = Release|Any CPU {764D2C19-0187-4837-A2A3-96DDC6EF4CE2}.Release|Any CPU.Build.0 = Release|Any CPU + {9860FCF7-3111-4C12-A16F-ACEBA42D930F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9860FCF7-3111-4C12-A16F-ACEBA42D930F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9860FCF7-3111-4C12-A16F-ACEBA42D930F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9860FCF7-3111-4C12-A16F-ACEBA42D930F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1168,6 +1174,7 @@ Global {B2B261E8-56EC-45DC-8AB8-A491A065EFD4} = {FD0FAF5F-1DED-485C-99FA-84B97F3A8EEC} {2F11618A-9251-4609-B3D5-CE4D2B3D3E49} = {5CA5F70E-0FDB-467B-B22C-3CD5994F0087} {764D2C19-0187-4837-A2A3-96DDC6EF4CE2} = {CC126D03-7EAC-493F-B187-DCDEE1EF6A70} + {9860FCF7-3111-4C12-A16F-ACEBA42D930F} = {FD0FAF5F-1DED-485C-99FA-84B97F3A8EEC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {604E6B91-7BC0-4126-AE07-D4D2FEFC3D29} diff --git a/build/Targets/Packages.props b/build/Targets/Packages.props index 7803ec7645c48f030136fe9470858f5f3b5301e3..4459bf6c8a69de01d0d2f9e551327ae9914407c4 100644 --- a/build/Targets/Packages.props +++ b/build/Targets/Packages.props @@ -11,6 +11,7 @@ 2.6.2-beta2-63202-01 0.9.3 + 0.11.0 8.0.1 8.0.0 2.3.1 diff --git a/src/Compilers/CSharp/Portable/Microsoft.CodeAnalysis.CSharp.csproj b/src/Compilers/CSharp/Portable/Microsoft.CodeAnalysis.CSharp.csproj index 1bba95ac42de47659bd342ee620dba8bab8376f6..66859f73ef2d2abe99545abbdfde6926fd54aac9 100644 --- a/src/Compilers/CSharp/Portable/Microsoft.CodeAnalysis.CSharp.csproj +++ b/src/Compilers/CSharp/Portable/Microsoft.CodeAnalysis.CSharp.csproj @@ -84,6 +84,7 @@ + \ No newline at end of file diff --git a/src/Compilers/CSharp/Test/Emit/Perf.cs b/src/Compilers/CSharp/Test/Emit/Perf.cs index 27ab88fa249be1603884de9c82653d4e3832b062..029536ab42515260ae74f53eec1c78e57369187a 100644 --- a/src/Compilers/CSharp/Test/Emit/Perf.cs +++ b/src/Compilers/CSharp/Test/Emit/Perf.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Microsoft.CodeAnalysis.CSharp.Test.Utilities; -using Microsoft.CodeAnalysis.Test.Utilities; using Xunit; namespace Microsoft.CodeAnalysis.CSharp.UnitTests.Emit diff --git a/src/Compilers/Core/Portable/Microsoft.CodeAnalysis.csproj b/src/Compilers/Core/Portable/Microsoft.CodeAnalysis.csproj index 1d178cfe9950ffb8a67e9e9123f5ab34772ebdec..888cf1dab1ae4974133927752a0fedffb7337a93 100644 --- a/src/Compilers/Core/Portable/Microsoft.CodeAnalysis.csproj +++ b/src/Compilers/Core/Portable/Microsoft.CodeAnalysis.csproj @@ -103,6 +103,7 @@ + diff --git a/src/Compilers/Test/Utilities/CSharp/Microsoft.CodeAnalysis.CSharp.Test.Utilities.csproj b/src/Compilers/Test/Utilities/CSharp/Microsoft.CodeAnalysis.CSharp.Test.Utilities.csproj index d90eef83a8fb7dc22ac48a4144ccc9cecde5036d..ab49afea996ad83f90d5711f046f0e7aa57e1f14 100644 --- a/src/Compilers/Test/Utilities/CSharp/Microsoft.CodeAnalysis.CSharp.Test.Utilities.csproj +++ b/src/Compilers/Test/Utilities/CSharp/Microsoft.CodeAnalysis.CSharp.Test.Utilities.csproj @@ -35,6 +35,7 @@ + diff --git a/src/Tools/CompilerBenchmarks/CompilerBenchmarks.csproj b/src/Tools/CompilerBenchmarks/CompilerBenchmarks.csproj new file mode 100644 index 0000000000000000000000000000000000000000..60724f6e0282adfcd08dacbb869953e2d7c79500 --- /dev/null +++ b/src/Tools/CompilerBenchmarks/CompilerBenchmarks.csproj @@ -0,0 +1,30 @@ + + + + + + Exe + netcoreapp2.0 + false + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Tools/CompilerBenchmarks/EmitBenchmark.cs b/src/Tools/CompilerBenchmarks/EmitBenchmark.cs new file mode 100644 index 0000000000000000000000000000000000000000..254fcf707edf6a0ab4c158dd40c540cc1037da36 --- /dev/null +++ b/src/Tools/CompilerBenchmarks/EmitBenchmark.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using BenchmarkDotNet.Attributes; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Emit; +using Roslyn.Utilities; +using static Microsoft.CodeAnalysis.Compilation; + +namespace CompilerBenchmarks +{ + public class EmitBenchmark + { + public enum EmitStageSelection + { + FullEmit, // Measures Compilation.Emit + SerializeOnly // Measures just metadata serialization (internal API) + } + + [Params(EmitStageSelection.FullEmit, EmitStageSelection.SerializeOnly)] + public EmitStageSelection Selection { get; set; } + + private Compilation _comp; + private CommonPEModuleBuilder _moduleBeingBuilt; + private EmitOptions _options; + private MemoryStream _peStream; + + [GlobalSetup] + public void GlobalSetup() + { + _peStream = new MemoryStream(); + _comp = Helpers.CreateReproCompilation(); + + // Call GetDiagnostics to force binding to finish and most semantic analysis to be completed + _ = _comp.GetDiagnostics(); + + if (Selection == EmitStageSelection.SerializeOnly) + { + _options = EmitOptions.Default.WithIncludePrivateMembers(true); + + bool embedPdb = _options.DebugInformationFormat == DebugInformationFormat.Embedded; + + var diagnostics = DiagnosticBag.GetInstance(); + + _moduleBeingBuilt = _comp.CheckOptionsAndCreateModuleBuilder( + diagnostics, + manifestResources: null, + _options, + debugEntryPoint: null, + sourceLinkStream: null, + embeddedTexts: null, + testData: null, + cancellationToken: default); + + bool success = false; + + success = _comp.CompileMethods( + _moduleBeingBuilt, + emittingPdb: embedPdb, + emitMetadataOnly: _options.EmitMetadataOnly, + emitTestCoverageData: _options.EmitTestCoverageData, + diagnostics: diagnostics, + filterOpt: null, + cancellationToken: default); + + _comp.GenerateResourcesAndDocumentationComments( + _moduleBeingBuilt, + xmlDocumentationStream: null, + win32ResourcesStream: null, + _options.OutputNameOverride, + diagnostics, + cancellationToken: default); + + _comp.ReportUnusedImports(null, diagnostics, default); + _moduleBeingBuilt.CompilationFinished(); + + diagnostics.Free(); + } + } + + [Benchmark] + public object RunEmit() + { + _peStream.Position = 0; + switch (Selection) + { + case EmitStageSelection.FullEmit: + { + return _comp.Emit(_peStream); + } + case EmitStageSelection.SerializeOnly: + { + var diagnostics = DiagnosticBag.GetInstance(); + + _comp.SerializeToPeStream( + _moduleBeingBuilt, + new SimpleEmitStreamProvider(_peStream), + metadataPEStreamProvider: null, + pdbStreamProvider: null, + testSymWriterFactory: null, + diagnostics, + metadataOnly: _options.EmitMetadataOnly, + includePrivateMembers: _options.IncludePrivateMembers, + emitTestCoverageData: _options.EmitTestCoverageData, + pePdbFilePath: _options.PdbFilePath, + privateKeyOpt: null, + cancellationToken: default); + + diagnostics.Free(); + + return _peStream; + } + + default: + throw ExceptionUtilities.UnexpectedValue(Selection); + } + } + } +} diff --git a/src/Tools/CompilerBenchmarks/FixedCsProjGenerator.cs b/src/Tools/CompilerBenchmarks/FixedCsProjGenerator.cs new file mode 100644 index 0000000000000000000000000000000000000000..c7b7117f2f50823d661c4b1736ba8def510d9cc6 --- /dev/null +++ b/src/Tools/CompilerBenchmarks/FixedCsProjGenerator.cs @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Text; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace CompilerBenchmarks +{ + public class FixedCsProjGenerator : CsProjGenerator + { + private const string Template = @" + + + false + false + + + + + + $PROGRAMNAME$ + $TFM$ + true + $PLATFORM$ + $PROGRAMNAME$ + Exe + bin\$CONFIGURATIONNAME$ + False + pdbonly + true + $COPIEDSETTINGS$ + + + + + + + + + + $RUNTIMESETTINGS$ + + +"; + + public FixedCsProjGenerator(string targetFrameworkMoniker, Func platformProvider, string runtimeFrameworkVersion = null) + : base(targetFrameworkMoniker, platformProvider, runtimeFrameworkVersion) + { } + + protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPartition, string programName) + { + string directoryName = Path.GetDirectoryName(buildPartition.AssemblyLocation) + ?? throw new DirectoryNotFoundException(buildPartition.AssemblyLocation); + return Path.Combine(directoryName, programName); + } + + public new static string PlatformProvider(Platform p) => p.ToString(); + + public static IToolchain Default => new Toolchain( + "FixedCsProjToolchain", + new FixedCsProjGenerator("netcoreapp2.0", PlatformProvider), + new DotNetCliBuilder("netcoreapp2.0"), + new DotNetCliExecutor(null)); + + protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) + { + string template = Template; + var benchmark = buildPartition.RepresentativeBenchmarkCase; + var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger); + + string platform = PlatformProvider(buildPartition.Platform); + string content = SetPlatform(template, platform); + content = SetCodeFileName(content, Path.GetFileName(artifactsPaths.ProgramCodePath)); + content = content.Replace("$CSPROJPATH$", projectFile.FullName); + content = SetTargetFrameworkMoniker(content, TargetFrameworkMoniker); + content = content.Replace("$PROGRAMNAME$", artifactsPaths.ProgramName); + content = content.Replace("$RUNTIMESETTINGS$", GetRuntimeSettings(benchmark.Job.Environment.Gc, buildPartition.Resolver)); + content = content.Replace("$COPIEDSETTINGS$", GetSettingsThatNeedsToBeCopied(projectFile)); + content = content.Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration); + + File.WriteAllText(artifactsPaths.ProjectFilePath, content); + } + + // the host project or one of the .props file that it imports might contain some custom settings that needs to be copied, sth like + // 2.0.0-beta-001607-00 + // 2.0.0-beta-001607-00 + private string GetSettingsThatNeedsToBeCopied(FileInfo projectFile) + { + if (!string.IsNullOrEmpty(RuntimeFrameworkVersion)) // some power users knows what to configure, just do it and copy nothing more + return $"{RuntimeFrameworkVersion}"; + + var customSettings = new StringBuilder(); + using (var file = new StreamReader(File.OpenRead(projectFile.FullName))) + { + string line; + while ((line = file.ReadLine()) != null) + { + if (line.Contains("NetCoreAppImplicitPackageVersion") || line.Contains("RuntimeFrameworkVersion") || line.Contains("PackageTargetFallback") || line.Contains("LangVersion")) + { + customSettings.Append(line); + } + else if (line.Contains(" + var directoryName = projectFile.DirectoryName ?? throw new DirectoryNotFoundException(projectFile.DirectoryName); + string absolutePath = File.Exists(propsFilePath) + ? propsFilePath // absolute path or relative to current dir + : Path.Combine(directoryName, propsFilePath); // relative to csproj + + if (File.Exists(absolutePath)) + { + customSettings.Append(GetSettingsThatNeedsToBeCopied(new FileInfo(absolutePath))); + } + } + } + } + + return customSettings.ToString(); + } + } +} diff --git a/src/Tools/CompilerBenchmarks/Helpers.cs b/src/Tools/CompilerBenchmarks/Helpers.cs new file mode 100644 index 0000000000000000000000000000000000000000..6f10d12a63ba72440af0aef8e403bbd269015b0b --- /dev/null +++ b/src/Tools/CompilerBenchmarks/Helpers.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Test.Utilities; + +namespace CompilerBenchmarks +{ + internal static class Helpers + { + public const string TestProjectEnvVarName = "ROSLYN_TEST_PROJECT_DIR"; + + public static Compilation CreateReproCompilation() + { + var projectDir = Environment.GetEnvironmentVariable(TestProjectEnvVarName); + var cmdLineParser = new CSharpCommandLineParser(); + var responseFile = Path.Combine(projectDir, "repro.rsp"); + var compiler = new MockCSharpCompiler(responseFile, projectDir, Array.Empty()); + var output = new StringWriter(); + return compiler.CreateCompilation(output, null, null); + } + } +} diff --git a/src/Tools/CompilerBenchmarks/Program.cs b/src/Tools/CompilerBenchmarks/Program.cs new file mode 100644 index 0000000000000000000000000000000000000000..fadb950d7339c85d88c6a736bb0b7e95d4fb2d1b --- /dev/null +++ b/src/Tools/CompilerBenchmarks/Program.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; + +namespace CompilerBenchmarks +{ + public class Program + { + private class IgnoreReleaseOnly : ManualConfig + { + public IgnoreReleaseOnly() + { + Add(JitOptimizationsValidator.DontFailOnError); + Add(DefaultConfig.Instance.GetLoggers().ToArray()); + Add(DefaultConfig.Instance.GetExporters().ToArray()); + Add(DefaultConfig.Instance.GetColumnProviders().ToArray()); + Add(new Job { Infrastructure = { Toolchain = FixedCsProjGenerator.Default } }); + } + } + + public static void Main(string[] args) + { + var projectPath = args[0]; + var artifactsPath = Path.Combine(projectPath, "../BenchmarkDotNet.Artifacts"); + + var config = new IgnoreReleaseOnly(); + var artifactsDir = Directory.CreateDirectory(artifactsPath); + config.ArtifactsPath = artifactsDir.FullName; + + // Benchmark.NET creates a new process to run the benchmark, so the easiest way + // to communicate information is pass by environment variable + Environment.SetEnvironmentVariable(Helpers.TestProjectEnvVarName, projectPath); + _ = BenchmarkRunner.Run(config); + } + } +} diff --git a/src/Tools/CompilerBenchmarks/README.md b/src/Tools/CompilerBenchmarks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a4a49944e78e47e6f012969a095d96e4f4eca52e --- /dev/null +++ b/src/Tools/CompilerBenchmarks/README.md @@ -0,0 +1,11 @@ + +This project contains a set of "micro" benchmarks for the compiler, focused on measuring +the time spent in specific phases of the compiler, e.g. Emit, metadata serialization, binding, +parsing, etc. + +To run all benchmarks, simply run the `run-perf.ps1` file on your machine, which should produce +a simple output table containing the results. To compare the results of your change, you can +run the script before and after and attempt to compare the results. Calculating statistical +significance is beyond the scope of this document, but you can get a general idea of whether +or not your changes are significant if the different is substantially larger than the "error" +value reported in the results summary. \ No newline at end of file diff --git a/src/Tools/CompilerBenchmarks/run-perf.ps1 b/src/Tools/CompilerBenchmarks/run-perf.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..9fc184bb761607a97ce1c89b755047c5b0b0e1ac --- /dev/null +++ b/src/Tools/CompilerBenchmarks/run-perf.ps1 @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +[CmdletBinding(PositionalBinding=$false)] +param ( + # By default, the Roslyn dir is expected to be next to this dir + [string]$roslynDir = "$PSScriptRoot/../../.." +) + +Set-Variable -Name LastExitCode 0 +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +try { + . (Join-Path $roslynDir "build/scripts/build-utils.ps1") + + # Download dotnet if it isn't already available + Ensure-DotnetSdk + + $reproPath = Join-Path $binariesDir "CodeAnalysisRepro" + + if (-not (Test-Path $reproPath)) { + $tmpFile = [System.IO.Path]::GetTempFileName() + Invoke-WebRequest -Uri "https://roslyninfra.blob.core.windows.net/perf-artifacts/CodeAnalysisRepro.zip" -UseBasicParsing -OutFile $tmpFile + [Reflection.Assembly]::LoadWithPartialName('System.IO.Compression.FileSystem') | Out-Null + [IO.Compression.ZipFile]::ExtractToDirectory($tmpFile, $binariesDir) + } + + Exec-Command "dotnet" "run -c Release $reproPath" +} +catch { + Write-Host $_ + Write-Host $_.Exception + exit 1 +}