From bdbd644c27dfc041eebfd2bbb009059258f7a9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moreno?= Date: Tue, 5 Jan 2021 09:40:42 +0100 Subject: [PATCH] parallelize hygiene --- build/azure-pipelines/product-build.yml | 6 -- build/azure-pipelines/product-compile.yml | 12 ++- build/azure-pipelines/product-hygiene.yml | 107 ---------------------- package.json | 1 + 4 files changed, 8 insertions(+), 118 deletions(-) delete mode 100644 build/azure-pipelines/product-hygiene.yml diff --git a/build/azure-pipelines/product-build.yml b/build/azure-pipelines/product-build.yml index a93b7897e4c..84afbc4b0bf 100644 --- a/build/azure-pipelines/product-build.yml +++ b/build/azure-pipelines/product-build.yml @@ -35,12 +35,6 @@ stages: VSCODE_ARCH: x64 steps: - template: product-compile.yml - - job: Hygiene - pool: tryout - variables: - VSCODE_ARCH: x64 - steps: - - template: product-hygiene.yml - stage: Windows dependsOn: diff --git a/build/azure-pipelines/product-compile.yml b/build/azure-pipelines/product-compile.yml index ba5e76ec5df..2737ea1d1f7 100644 --- a/build/azure-pipelines/product-compile.yml +++ b/build/azure-pipelines/product-compile.yml @@ -87,15 +87,17 @@ steps: - script: | set -e - yarn npm-run-all -lp compile-build compile-extensions-build + yarn npm-run-all -lp hygiene monaco-compile-check valid-layers-check compile-build compile-extensions-build yarn npm-run-all -lp minify-vscode minify-vscode-reh minify-vscode-reh-web - displayName: Compile + displayName: Hygiene & Compile - script: | set -e - AZURE_STORAGE_ACCESS_KEY="$(ticino-storage-key)" \ - node build/azure-pipelines/upload-sourcemaps - displayName: Upload sourcemaps + AZURE_STORAGE_ACCESS_KEY="$(ticino-storage-key)" node build/azure-pipelines/upload-sourcemaps & + ./build/azure-pipelines/common/extract-telemetry.sh & + AZURE_WEBVIEW_STORAGE_ACCESS_KEY="$(vscode-webview-storage-key)" ./build/azure-pipelines/common/publish-webview.sh & + wait + displayName: Publish compilation assets condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false')) - script: | diff --git a/build/azure-pipelines/product-hygiene.yml b/build/azure-pipelines/product-hygiene.yml deleted file mode 100644 index 477336cd536..00000000000 --- a/build/azure-pipelines/product-hygiene.yml +++ /dev/null @@ -1,107 +0,0 @@ -steps: - - task: NodeTool@0 - inputs: - versionSpec: "12.x" - - - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@2 - inputs: - versionSpec: "1.x" - - - task: AzureKeyVault@1 - displayName: "Azure Key Vault: Get Secrets" - inputs: - azureSubscription: "vscode-builds-subscription" - KeyVaultName: vscode - - - script: | - set -e - cat << EOF > ~/.netrc - machine github.com - login vscode - password $(github-distro-mixin-password) - EOF - - git config user.email "vscode@microsoft.com" - git config user.name "VSCode" - displayName: Prepare tooling - - - script: | - set -e - git pull --no-rebase https://github.com/$(VSCODE_MIXIN_REPO).git $(node -p "require('./package.json').distro") - displayName: Merge distro - - - script: | - npx https://aka.ms/enablesecurefeed standAlone - displayName: Switch to Terrapin packages - timeoutInMinutes: 5 - condition: and(succeeded(), eq(variables['ENABLE_TERRAPIN'], 'true')) - - - script: | - mkdir -p .build - node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash - displayName: Prepare yarn cache flags - - # using `genericNodeModules` instead of `nodeModules` here to avoid sharing the cache with builds running inside containers - - task: Cache@2 - inputs: - key: 'genericNodeModules | $(Agent.OS) | .build/yarnlockhash' - path: .build/node_modules_cache - cacheHitVar: NODE_MODULES_RESTORED - displayName: Cache node_modules archive - - - script: | - set -e - tar -xzf .build/node_modules_cache/cache.tgz - condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) - displayName: Extract node_modules archive - - - script: | - set -e - for i in {1..3}; do # try 3 times, for Terrapin - yarn --frozen-lockfile && break - if [ $i -eq 3 ]; then - echo "Yarn failed too many times" >&2 - exit 1 - fi - echo "Yarn failed $i, trying again..." - done - env: - ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - displayName: Install dependencies - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - - script: | - set -e - node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt - mkdir -p .build/node_modules_cache - tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - displayName: Create node_modules archive - - # Mixin must run before optimize, because the CSS loader will inline small SVGs - - script: | - set -e - node build/azure-pipelines/mixin - displayName: Mix in quality - - - script: | - set -e - yarn gulp hygiene - yarn monaco-compile-check - yarn valid-layers-check - displayName: Run hygiene, monaco compile & valid layers checks - condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - - - script: | - set - - ./build/azure-pipelines/common/extract-telemetry.sh - displayName: Extract Telemetry - condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false')) - - - script: | - set -e - AZURE_WEBVIEW_STORAGE_ACCESS_KEY="$(vscode-webview-storage-key)" \ - ./build/azure-pipelines/common/publish-webview.sh - displayName: Publish Webview - condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false')) diff --git a/package.json b/package.json index 56703cd3dc8..443db8e6122 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "watch-extensionsd": "deemon yarn watch-extensions", "kill-watch-extensionsd": "deemon --kill yarn watch-extensions", "mocha": "mocha test/unit/node/all.js --delay", + "hygiene": "node --max_old_space_size=4095 ./node_modules/gulp/bin/gulp.js hygiene", "precommit": "node build/hygiene.js", "gulp": "node --max_old_space_size=8192 ./node_modules/gulp/bin/gulp.js", "electron": "node build/lib/electron", -- GitLab