From 05e2cfbc6b0ec1495495ca22f4256917456c8719 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Wed, 5 Aug 2020 14:38:10 -0700 Subject: [PATCH] Add wasm runs (#39548) * Add WASM microbenchmark runs This commit adds WASM microbenchmark support to the perf lab. * Increase timeout for first run * Add runkind * Update proj file selection --- eng/common/performance/microbenchmarks.proj | 6 +++- eng/common/performance/performance-setup.sh | 31 ++++++++++++++++--- eng/pipelines/coreclr/perf.yml | 30 ++++++++++++++++++ eng/pipelines/coreclr/templates/perf-job.yml | 19 +++++++++++- .../coreclr/templates/run-performance-job.yml | 4 +-- 5 files changed, 82 insertions(+), 8 deletions(-) diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj index 185522fc739..71114b074e1 100644 --- a/eng/common/performance/microbenchmarks.proj +++ b/eng/common/performance/microbenchmarks.proj @@ -41,6 +41,10 @@ $HELIX_WORKITEM_ROOT/testResults.xml + + $(CliArguments) --wasm + + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe @@ -65,7 +69,7 @@ - 2:30 + 12:30 0:15 diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index c87cbf0fc23..66f63a397a0 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -24,6 +24,8 @@ run_from_perf_repo=false use_core_run=true use_baseline_core_run=true using_mono=false +wasm_runtime_loc= +using_wasm=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -70,7 +72,7 @@ while (($# > 0)); do ;; --kind) kind=$2 - configurations="CompliationMode=$compilation_mode RunKind=$kind" + configurations="CompilationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) @@ -101,6 +103,10 @@ while (($# > 0)); do mono_dotnet=$2 shift 2 ;; + --wasm) + wasm_runtime_loc=$2 + shift 2 + ;; --compare) compare=true shift 1 @@ -130,6 +136,7 @@ while (($# > 0)); do echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." + echo " --wasm Path to the unpacled wasm runtime pack." echo "" exit 0 ;; @@ -141,7 +148,7 @@ if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-perfo fi if [ -z "$configurations" ]; then - configurations="CompliationMode=$compilation_mode" + configurations="CompilationMode=$compilation_mode" fi if [ -z "$core_root_directory" ]; then @@ -191,11 +198,15 @@ if [[ "$mono_dotnet" != "" ]]; then configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" fi +if [[ "$wasm_runtime_loc" != "" ]]; then + configurations="CompilationMode=wasm;RunKind=micro" +fi + if [[ "$monointerpreter" == "true" ]]; then - extra_benchmark_dotnet_arguments="--category-exclusion-filter NoInterpreter" + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter" fi -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" +common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs \"$configurations\" --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" @@ -217,6 +228,17 @@ else mv $docs_directory $workitem_directory fi +if [[ "$wasm_runtime_loc" != "" ]]; then + using_wasm=true + + wasm_dotnet_path=$payload_directory/dotnet-wasm + + mv $wasm_runtime_loc $wasm_dotnet_path + + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm" + +fi + if [[ "$mono_dotnet" != "" ]]; then using_mono=true mono_dotnet_path=$payload_directory/dotnet-mono @@ -259,3 +281,4 @@ Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable fal Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false +Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index cb97c408514..0cbf812e809 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -56,6 +56,21 @@ jobs: testGroup: perf # build mono +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Browser_wasm + jobParameters: + buildArgs: -s mono+libs+installer -c $(_BuildConfig) + nameSuffix: wasm + isOfficialBuild: ${{ variables.isOfficialBuild }} + extraStepsTemplate: /eng/pipelines/common/upload-unsigned-artifacts-step.yml + extraStepsParameters: + name: MonoRuntimePacks + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/mono/templates/build-job.yml @@ -97,6 +112,21 @@ jobs: runKind: micro_mono # run coreclr microbenchmarks perf job +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: wasm + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: wasm + codeGenType: 'wasm' + projectFile: microbenchmarks.proj + runKind: micro + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 57df1c0dcb6..21799d64354 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -43,6 +43,8 @@ jobs: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - ${{ if eq(parameters.runtimeType, 'mono') }}: - ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if eq(parameters.runtimeType, 'wasm') }}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', 'Browser', '', 'wasm', parameters.buildConfig, parameters.runtimeType) }} ${{ if eq(parameters.osGroup, 'Windows_NT') }}: ${{ if eq(parameters.runtimeType, 'mono') }}: @@ -52,7 +54,9 @@ jobs: ${{ if ne(parameters.osGroup, 'Windows_NT') }}: ${{ if eq(parameters.runtimeType, 'mono') }}: extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono --kind micro_mono - ${{ if ne(parameters.runtimeType, 'mono') }}: + ${{ if eq(parameters.runtimeType, 'wasm') }}: + extraSetupParameters: --architecture ${{ parameters.archType }} --wasm $(librariesDownloadDir)/bin/wasm --kind micro + ${{ if eq(parameters.runtimeType, 'coreclr') }}: extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} variables: ${{ parameters.variables }} @@ -90,6 +94,19 @@ jobs: artifactName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)' displayName: 'Mono runtime' + - ${{ if eq(parameters.runtimeType, 'wasm') }}: + # Download artifact + - task: DownloadBuildArtifacts@0 + displayName: 'Download WASM Runtime Pack' + inputs: + buildType: current + downloadType: single + downloadPath: '$(Build.SourcesDirectory)/__download__' + artifactName: 'IntermediateUnsignedArtifacts' + + - script: "mkdir $(librariesDownloadDir)/bin/wasm;unzip -o $(Build.SourcesDirectory)/__download__/IntermediateUnsignedArtifacts/MonoRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.browser-wasm.5.0.0-ci.nupkg data/* runtimes/* -d $(librariesDownloadDir)/bin/wasm;cp src/mono/wasm/runtime-test.js $(librariesDownloadDir)/bin/wasm/runtime-test.js;find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \\;" + displayName: "Create wasm directory (Linux)" + # Create Core_Root - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) displayName: Create Core_Root diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index 3d34f5a67a1..0e44fbb7cd2 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -53,7 +53,7 @@ jobs: - IsInternal: '' - HelixApiAccessToken: '' - HelixPreCommandStemWindows: 'py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install --user azure.storage.blob==12.0.0 --force-reinstall;py -3 -m pip install --user azure.storage.queue==12.0.0 --force-reinstall;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' - - HelixPreCommandStemLinux: 'sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/Scripts/activate;export PYTHONPATH=;pip3 install --user azure.storage.blob==12.0.0 --force-reinstall;pip3 install --user azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' + - HelixPreCommandStemLinux: 'sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/Scripts/activate;export PYTHONPATH=;pip3 install --user azure.storage.blob==12.0.0 --force-reinstall;pip3 install --user azure.storage.queue==12.0.0 --force-reinstall;npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g;$HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - ExtraMSBuildLogsWindows: 'set MSBUILDDEBUGCOMM=1;set "MSBUILDDEBUGPATH=%HELIX_WORKITEM_UPLOAD_ROOT%"' - ExtraMSBuildLogsLinux: 'export MSBUILDDEBUGCOMM=1;export "MSBUILDDEBUGPATH=$HELIX_WORKITEM_UPLOAD_ROOT"' - HelixPreCommand: '' @@ -73,7 +73,7 @@ jobs: - ${{ if eq( parameters.osGroup, 'Windows_NT') }}: - HelixPreCommand: $(ExtraMSBuildLogsWindows) - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - HelixPreCommand: $(ExtraMSBuildLogsLinux) + - HelixPreCommand: $(ExtraMSBuildLogsLinux);npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g;$HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8 - ${{ if and(eq(parameters.codeGenType, 'Interpreter'), eq(parameters.runtimeType, 'mono')) }}: -- GitLab