Improvements to Telemetry Extractor Build (#76196)

* Allow the error to be surfaced to the user to provide more information about what is happening

* Testing gulp src

* Move json to app root

* Telemetry tooling commit update

* Harcode telemetry commit

* Update distro to match master

* statSync try catch

* build 💄

* fix build

* make build fail

* fix build

* fix build

* whitespace

build/azure-pipelines/darwin/product-build-darwin.yml

@@ -61,17 +61,17 @@ steps:
 
 - script: |
     set -e
-    cd ..
+    cd $BUILD_STAGINGDIRECTORY
     git clone https://github.com/microsoft/vscode-telemetry-extractor.git
     cd vscode-telemetry-extractor
-    npm install
-    ./node_modules/typescript/bin/tsc
+    git checkout 3b04aba5bfdfcca1a5426cd2c51a90d18740d0bc
+    npm i
     npm run setup-extension-repos
-    node ./out/cli-extract.js --sourceDir ../s --excludedDirPattern extensions  --outputDir . --applyEndpoints --includeIsMeasurement --patchWebsiteEvents
-    node ./out/cli-extract-extensions.js --sourceDir ./src/telemetry-sources/ --outputDir . --applyEndpoints --includeIsMeasurement
-    mv declarations-resolved.json ../s/src/telemetry-core.json
-    mv declarations-extensions-resolved.json ../s/src/telemetry-extensions.json
-    echo 'Moved Files'
+    node ./out/cli-extract.js --sourceDir $BUILD_SOURCESDIRECTORY --excludedDirPattern extensions  --outputDir . --applyEndpoints --includeIsMeasurement --patchWebsiteEvents
+    node ./out/cli-extract-extensions.js --sourceDir ./src/telemetry-sources --outputDir . --applyEndpoints --includeIsMeasurement
+    mkdir -p $BUILD_SOURCESDIRECTORY/.build/telemetry
+    mv declarations-resolved.json $BUILD_SOURCESDIRECTORY/.build/telemetry/telemetry-core.json
+    mv declarations-extensions-resolved.json $BUILD_SOURCESDIRECTORY/.build/telemetry/telemetry-extensions.json
   displayName: Extract Telemetry
 
 - script: |

build/azure-pipelines/linux/product-build-linux.yml

@@ -62,17 +62,17 @@ steps:
 
 - script: |
     set -e
-    cd ..
+    cd $BUILD_STAGINGDIRECTORY
     git clone https://github.com/microsoft/vscode-telemetry-extractor.git
     cd vscode-telemetry-extractor
-    npm install
-    ./node_modules/typescript/bin/tsc
+    git checkout 3b04aba5bfdfcca1a5426cd2c51a90d18740d0bc
+    npm i
     npm run setup-extension-repos
-    node ./out/cli-extract.js --sourceDir ../s --excludedDirPattern extensions  --outputDir . --applyEndpoints --includeIsMeasurement --patchWebsiteEvents
-    node ./out/cli-extract-extensions.js --sourceDir ./src/telemetry-sources/ --outputDir . --applyEndpoints --includeIsMeasurement
-    mv declarations-resolved.json ../s/src/telemetry-core.json
-    mv declarations-extensions-resolved.json ../s/src/telemetry-extensions.json
-    echo 'Moved Files'
+    node ./out/cli-extract.js --sourceDir $BUILD_SOURCESDIRECTORY --excludedDirPattern extensions  --outputDir . --applyEndpoints --includeIsMeasurement --patchWebsiteEvents
+    node ./out/cli-extract-extensions.js --sourceDir ./src/telemetry-sources --outputDir . --applyEndpoints --includeIsMeasurement
+    mkdir -p $BUILD_SOURCESDIRECTORY/.build/telemetry
+    mv declarations-resolved.json $BUILD_SOURCESDIRECTORY/.build/telemetry/telemetry-core.json
+    mv declarations-extensions-resolved.json $BUILD_SOURCESDIRECTORY/.build/telemetry/telemetry-extensions.json
   displayName: Extract Telemetry
 
 - script: |

build/azure-pipelines/win32/product-build-win32.yml

@@ -68,18 +68,17 @@ steps:
 - powershell: |
     . build/azure-pipelines/win32/exec.ps1
     $ErrorActionPreference = "Stop"
-    cd ..
+    cd $env:BUILD_STAGINGDIRECTORY
     git clone https://github.com/microsoft/vscode-telemetry-extractor.git
     cd vscode-telemetry-extractor
-    npm install
-    npm install -g typescript
-    tsc
+    git checkout 3b04aba5bfdfcca1a5426cd2c51a90d18740d0bc
+    npm i
     npm run setup-extension-repos
-    node ./out/cli-extract.js --sourceDir ../s --excludedDirPattern extensions  --outputDir . --applyEndpoints --includeIsMeasurement --patchWebsiteEvents
-    node ./out/cli-extract-extensions.js --sourceDir ./src/telemetry-sources/ --outputDir . --applyEndpoints --includeIsMeasurement
-    mv declarations-resolved.json ../s/src/telemetry-core.json
-    mv declarations-extensions-resolved.json ../s/src/telemetry-extensions.json
-    echo 'Moved Files'
+    node .\out\cli-extract.js --sourceDir $env:BUILD_SOURCESDIRECTORY --excludedDirPattern extensions  --outputDir . --applyEndpoints --includeIsMeasurement --patchWebsiteEvents
+    node .\out\cli-extract-extensions.js --sourceDir .\src\telemetry-sources --outputDir . --applyEndpoints --includeIsMeasurement
+    mkdir $env:BUILD_SOURCESDIRECTORY\.build\telemetry -ea 0
+    mv declarations-resolved.json $env:BUILD_SOURCESDIRECTORY\.build\telemetry\telemetry-core.json
+    mv declarations-extensions-resolved.json $env:BUILD_SOURCESDIRECTORY\.build\telemetry\telemetry-extensions.json
   displayName: Extract Telemetry
 
 - powershell: |

build/gulpfile.vscode.js

@@ -62,8 +62,6 @@ const vscodeResources = [
 	'out-build/bootstrap-amd.js',
 	'out-build/bootstrap-window.js',
 	'out-build/paths.js',
-	'out-build/telemetry-core.json',
-	'out-build/telemetry-extensions.json',
 	'out-build/vs/**/*.{svg,png,cur,html}',
 	'!out-build/vs/code/browser/**/*.html',
 	'out-build/vs/base/common/performance.js',
@@ -317,6 +315,8 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 		// TODO the API should be copied to `out` during compile, not here
 		const api = gulp.src('src/vs/vscode.d.ts').pipe(rename('out/vs/vscode.d.ts'));
 
+		const telemetry = gulp.src('.build/telemetry/**', { base: '.build/telemetry', dot: true });
+
 		const depsSrc = [
 			..._.flatten(productionDependencies.map(d => path.relative(root, d.path)).map(d => [`${d}/**`, `!${d}/**/{test,tests}/**`])),
 			// @ts-ignore JSON checking: dependencies is optional
@@ -333,6 +333,7 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 			productJsonStream,
 			license,
 			api,
+			telemetry,
 			sources,
 			deps
 		);

src/vs/platform/environment/node/argv.ts

@@ -220,35 +220,38 @@ export function buildVersionMessage(version: string | undefined, commit: string
 }
 
 export function buildTelemetryMessage(appRoot: string, extensionsPath: string): string {
-	try {
-		// Gets all the directories inside the extension directory
-		const dirs = readdirSync(extensionsPath).filter(files => statSync(join(extensionsPath, files)).isDirectory());
-		const telemetryJsonFolders: string[] = [];
-		dirs.forEach((dir) => {
-			const files = readdirSync(join(extensionsPath, dir)).filter(file => file === 'telemetry.json');
-			// We know it contains a telemetry.json file so we add it to the list of folders which have one
-			if (files.length === 1) {
-				telemetryJsonFolders.push(dir);
-			}
-		});
-		const mergedTelemetry = Object.create(null);
-		// Simple function to merge the telemetry into one json object
-		const mergeTelemetry = (contents: string, dirName: string) => {
-			const telemetryData = JSON.parse(contents);
-			mergedTelemetry[dirName] = telemetryData;
-		};
-		telemetryJsonFolders.forEach((folder) => {
-			const contents = readFileSync(join(extensionsPath, folder, 'telemetry.json')).toString();
-			mergeTelemetry(contents, folder);
-		});
-		let contents = readFileSync(join(appRoot, 'out/', 'telemetry-core.json')).toString();
-		mergeTelemetry(contents, 'vscode-core');
-		contents = readFileSync(join(appRoot, 'out/', 'telemetry-extensions.json')).toString();
-		mergeTelemetry(contents, 'vscode-extensions');
-		return JSON.stringify(mergedTelemetry, null, 4);
-	} catch (err) {
-		return 'Unable to read VS Code telemetry events!';
-	}
+	// Gets all the directories inside the extension directory
+	const dirs = readdirSync(extensionsPath).filter(files => {
+		// This handles case where broken symbolic links can cause statSync to throw and error
+		try {
+			return statSync(join(extensionsPath, files)).isDirectory();
+		} catch {
+			return false;
+		}
+	});
+	const telemetryJsonFolders: string[] = [];
+	dirs.forEach((dir) => {
+		const files = readdirSync(join(extensionsPath, dir)).filter(file => file === 'telemetry.json');
+		// We know it contains a telemetry.json file so we add it to the list of folders which have one
+		if (files.length === 1) {
+			telemetryJsonFolders.push(dir);
+		}
+	});
+	const mergedTelemetry = Object.create(null);
+	// Simple function to merge the telemetry into one json object
+	const mergeTelemetry = (contents: string, dirName: string) => {
+		const telemetryData = JSON.parse(contents);
+		mergedTelemetry[dirName] = telemetryData;
+	};
+	telemetryJsonFolders.forEach((folder) => {
+		const contents = readFileSync(join(extensionsPath, folder, 'telemetry.json')).toString();
+		mergeTelemetry(contents, folder);
+	});
+	let contents = readFileSync(join(appRoot, 'telemetry-core.json')).toString();
+	mergeTelemetry(contents, 'vscode-core');
+	contents = readFileSync(join(appRoot, 'telemetry-extensions.json')).toString();
+	mergeTelemetry(contents, 'vscode-extensions');
+	return JSON.stringify(mergedTelemetry, null, 4);
 }
 
 /**