diff --git a/build/.cachesalt b/build/.cachesalt index cf053dd3b9b938af9c0fac1c4451be6642f1c1d0..3a737baaa4aadcb8cff54af675d4046ec5259490 100644 --- a/build/.cachesalt +++ b/build/.cachesalt @@ -1 +1 @@ -2020-12-17T07:59:24.060Z +2020-12-24T14:06:35Z diff --git a/build/azure-pipelines/common/listNodeModules.js b/build/azure-pipelines/common/listNodeModules.js new file mode 100644 index 0000000000000000000000000000000000000000..540569c79b558b761aacfe1776a876799c0847d1 --- /dev/null +++ b/build/azure-pipelines/common/listNodeModules.js @@ -0,0 +1,40 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; +Object.defineProperty(exports, "__esModule", { value: true }); +const fs = require("fs"); +const path = require("path"); +if (process.argv.length !== 3) { + console.error('Usage: node listNodeModules.js OUTPUT_FILE'); + process.exit(-1); +} +const ROOT = path.join(__dirname, '../../../'); +function findNodeModulesFiles(location, inNodeModules, result) { + const entries = fs.readdirSync(path.join(ROOT, location)); + for (const entry of entries) { + const entryPath = `${location}/${entry}`; + if (/(^\/out)|(^\/src$)|(^\/.git$)|(^\/.build$)/.test(entryPath)) { + continue; + } + let stat; + try { + stat = fs.statSync(path.join(ROOT, entryPath)); + } + catch (err) { + continue; + } + if (stat.isDirectory()) { + findNodeModulesFiles(entryPath, inNodeModules || (entry === 'node_modules'), result); + } + else { + if (inNodeModules) { + result.push(entryPath.substr(1)); + } + } + } +} +const result = []; +findNodeModulesFiles('', false, result); +fs.writeFileSync(process.argv[2], result.join('\n') + '\n'); diff --git a/build/azure-pipelines/common/listNodeModules.ts b/build/azure-pipelines/common/listNodeModules.ts new file mode 100644 index 0000000000000000000000000000000000000000..2ed6294477a9f591328376972861b1e96e7edaa7 --- /dev/null +++ b/build/azure-pipelines/common/listNodeModules.ts @@ -0,0 +1,46 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import * as fs from 'fs'; +import * as path from 'path'; + +if (process.argv.length !== 3) { + console.error('Usage: node listNodeModules.js OUTPUT_FILE'); + process.exit(-1); +} + +const ROOT = path.join(__dirname, '../../../'); + +function findNodeModulesFiles(location: string, inNodeModules: boolean, result: string[]) { + const entries = fs.readdirSync(path.join(ROOT, location)); + for (const entry of entries) { + const entryPath = `${location}/${entry}`; + + if (/(^\/out)|(^\/src$)|(^\/.git$)|(^\/.build$)/.test(entryPath)) { + continue; + } + + let stat: fs.Stats; + try { + stat = fs.statSync(path.join(ROOT, entryPath)); + } catch (err) { + continue; + } + + if (stat.isDirectory()) { + findNodeModulesFiles(entryPath, inNodeModules || (entry === 'node_modules'), result); + } else { + if (inNodeModules) { + result.push(entryPath.substr(1)); + } + } + } +} + +const result: string[] = []; +findNodeModulesFiles('', false, result); +fs.writeFileSync(process.argv[2], result.join('\n') + '\n'); diff --git a/build/azure-pipelines/darwin/product-build-darwin.yml b/build/azure-pipelines/darwin/product-build-darwin.yml index fc4f8741cd0510519ba3ae5ff3ee1266f8807e14..15dcff16ebffe2faa197e984e2c359c6b113ebaa 100644 --- a/build/azure-pipelines/darwin/product-build-darwin.yml +++ b/build/azure-pipelines/darwin/product-build-darwin.yml @@ -58,11 +58,18 @@ steps: node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - task: Cache@2 inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Cache node_modules archive + + - script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e @@ -88,14 +95,15 @@ steps: ELECTRON_SKIP_BINARY_DOWNLOAD: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 displayName: Install dependencies - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - script: | set -e @@ -166,7 +174,7 @@ steps: set -e yarn --cwd test/integration/browser compile displayName: Compile integration tests - condition: and(succeeded(), eq(variables['CacheRestored'], 'true'), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - script: | # Figure out the full absolute path of the product we just built @@ -205,7 +213,7 @@ steps: set -e yarn --cwd test/smoke compile displayName: Compile smoke tests - condition: and(succeeded(), eq(variables['CacheRestored'], 'true'), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - script: | set -e diff --git a/build/azure-pipelines/linux/product-build-alpine.yml b/build/azure-pipelines/linux/product-build-alpine.yml index 8ebcd313090f0f3ac1b2e86c03607435d0c01aa4..72e9840d09b18523b299cb99333f85ac5cc9289b 100644 --- a/build/azure-pipelines/linux/product-build-alpine.yml +++ b/build/azure-pipelines/linux/product-build-alpine.yml @@ -61,11 +61,18 @@ steps: node build/azure-pipelines/common/computeNodeModulesCacheKey.js "alpine" $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - task: Cache@2 inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Cache node_modules archive + + - script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e @@ -81,14 +88,15 @@ steps: ELECTRON_SKIP_BINARY_DOWNLOAD: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 displayName: Install dependencies - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - script: | set -e diff --git a/build/azure-pipelines/linux/product-build-linux.yml b/build/azure-pipelines/linux/product-build-linux.yml index 558d6573f1e97a029911cd73888a54f528b5f545..fc25e4ee01175c335314d8cfed21bd56c0e2db70 100644 --- a/build/azure-pipelines/linux/product-build-linux.yml +++ b/build/azure-pipelines/linux/product-build-linux.yml @@ -52,11 +52,18 @@ steps: node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - task: Cache@2 inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Cache node_modules archive + + - script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e @@ -92,14 +99,15 @@ steps: ELECTRON_SKIP_BINARY_DOWNLOAD: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 displayName: Install dependencies - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - script: | set -e @@ -144,7 +152,7 @@ steps: set -e yarn --cwd test/integration/browser compile displayName: Compile integration tests - condition: and(succeeded(), eq(variables['CacheRestored'], 'true'), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - script: | # Figure out the full absolute path of the product we just built diff --git a/build/azure-pipelines/product-compile.yml b/build/azure-pipelines/product-compile.yml index efbe6dfd60d66d27aae85218e878c2f296a1ff3f..749d9b67215ca052be9066476305c36639af2c45 100644 --- a/build/azure-pipelines/product-compile.yml +++ b/build/azure-pipelines/product-compile.yml @@ -41,11 +41,18 @@ steps: node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - task: Cache@2 inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Cache node_modules archive + + - script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e @@ -74,14 +81,15 @@ steps: ELECTRON_SKIP_BINARY_DOWNLOAD: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 displayName: Install dependencies - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive # Mixin must run before optimize, because the CSS loader will inline small SVGs - script: | diff --git a/build/azure-pipelines/product-hygiene.yml b/build/azure-pipelines/product-hygiene.yml index b776e277abe921d551d36283d9728d0651e9d468..40a6b1eeb9465952f1cd3552e5c37bf5154a2fb7 100644 --- a/build/azure-pipelines/product-hygiene.yml +++ b/build/azure-pipelines/product-hygiene.yml @@ -41,11 +41,18 @@ steps: node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - task: Cache@2 inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Cache node_modules archive + + - script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e @@ -61,7 +68,15 @@ steps: ELECTRON_SKIP_BINARY_DOWNLOAD: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 displayName: Install dependencies - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + + - script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive # Mixin must run before optimize, because the CSS loader will inline small SVGs - script: | diff --git a/build/azure-pipelines/web/product-build-web.yml b/build/azure-pipelines/web/product-build-web.yml index e7699fd52c503d8346c4bbfaaba5b54dc5ee2029..69898811c89c8169f9f63b2c8b0075c1d708170e 100644 --- a/build/azure-pipelines/web/product-build-web.yml +++ b/build/azure-pipelines/web/product-build-web.yml @@ -52,11 +52,18 @@ steps: node build/azure-pipelines/common/computeNodeModulesCacheKey.js "web" $ENABLE_TERRAPIN > .build/yarnlockhash displayName: Prepare yarn cache flags - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - task: Cache@2 inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Cache node_modules archive + + - script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e @@ -72,14 +79,15 @@ steps: ELECTRON_SKIP_BINARY_DOWNLOAD: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 displayName: Install dependencies - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - inputs: - keyfile: ".build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - script: | set -e diff --git a/build/azure-pipelines/win32/product-build-win32.yml b/build/azure-pipelines/win32/product-build-win32.yml index 48fb33a269f174437aae1ff9f8e0e5ec19bdf0b8..1b2b717abc8e3b0988f6f6913522d696cd5ea05a 100644 --- a/build/azure-pipelines/win32/product-build-win32.yml +++ b/build/azure-pipelines/win32/product-build-win32.yml @@ -57,11 +57,19 @@ steps: node build/azure-pipelines/common/computeNodeModulesCacheKey.js > .build/yarnlockhash displayName: Prepare yarn cache flags - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - task: Cache@2 inputs: - keyfile: ".build/arch, .build/terrapin, .build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" + key: 'nodeModules | $(Agent.OS) | .build/arch, .build/terrapin, .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Cache node_modules archive + + - powershell: | + . build/azure-pipelines/win32/exec.ps1 + $ErrorActionPreference = "Stop" + exec { 7z.exe x .build/node_modules_cache/cache.7z -aos } + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - powershell: | . build/azure-pipelines/win32/exec.ps1 @@ -74,14 +82,16 @@ steps: ELECTRON_SKIP_BINARY_DOWNLOAD: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 displayName: Install dependencies - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - inputs: - keyfile: ".build/arch, .build/terrapin, .build/yarnlockhash" - targetfolder: "**/node_modules, !**/node_modules/**/node_modules" - vstsFeed: "npm-vscode" - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - powershell: | + . build/azure-pipelines/win32/exec.ps1 + $ErrorActionPreference = "Stop" + exec { node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt } + exec { mkdir -Force .build/node_modules_cache } + exec { 7z.exe a .build/node_modules_cache/cache.7z -mx3 `@.build/node_modules_list.txt } + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - powershell: | . build/azure-pipelines/win32/exec.ps1 @@ -145,7 +155,7 @@ steps: $ErrorActionPreference = "Stop" exec { yarn --cwd test/integration/browser compile } displayName: Compile integration tests - condition: and(succeeded(), eq(variables['CacheRestored'], 'true'), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64')) + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64')) - powershell: | # Figure out the full absolute path of the product we just built