From 4cdb2dfa3cf45ab92bf8c6d93608685394d3ec46 Mon Sep 17 00:00:00 2001 From: SteVen Batten <6561887+sbatten@users.noreply.github.com> Date: Thu, 23 Jul 2020 09:28:50 -0700 Subject: [PATCH] TAS-based experiment service (#103177) * new experimentation service based on tas-client * fixes for exp service * add event classifications * leverage product.json --- package.json | 1 + .../standalone/browser/simpleServices.ts | 3 + .../platform/product/common/productService.ts | 7 + src/vs/platform/telemetry/common/telemetry.ts | 2 + .../telemetry/common/telemetryService.ts | 8 + .../telemetry/common/telemetryUtils.ts | 1 + .../experiment/common/experimentService.ts | 13 ++ .../electron-browser/experimentService.ts | 209 ++++++++++++++++++ .../telemetry/browser/telemetryService.ts | 4 + .../electron-browser/telemetryService.ts | 4 + .../textsearch.perf.integrationTest.ts | 3 + src/vs/workbench/workbench.desktop.main.ts | 1 + yarn.lock | 23 +- 13 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 src/vs/workbench/services/experiment/common/experimentService.ts create mode 100644 src/vs/workbench/services/experiment/electron-browser/experimentService.ts diff --git a/package.json b/package.json index 63036471803..70e50d71d95 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "semver-umd": "^5.5.7", "spdlog": "^0.11.1", "sudo-prompt": "9.1.1", + "tas-client": "^0.0.950", "v8-inspect-profiler": "^0.0.20", "vscode-nsfw": "1.2.8", "vscode-oniguruma": "1.3.1", diff --git a/src/vs/editor/standalone/browser/simpleServices.ts b/src/vs/editor/standalone/browser/simpleServices.ts index df5c7eb96b5..9051fb2cc78 100644 --- a/src/vs/editor/standalone/browser/simpleServices.ts +++ b/src/vs/editor/standalone/browser/simpleServices.ts @@ -552,6 +552,9 @@ export class StandaloneTelemetryService implements ITelemetryService { public setEnabled(value: boolean): void { } + public setExperimentProperty(name: string, value: string): void { + } + public publicLog(eventName: string, data?: any): Promise { return Promise.resolve(undefined); } diff --git a/src/vs/platform/product/common/productService.ts b/src/vs/platform/product/common/productService.ts index 188786627ce..b119d2b8048 100644 --- a/src/vs/platform/product/common/productService.ts +++ b/src/vs/platform/product/common/productService.ts @@ -49,6 +49,13 @@ export interface IProductConfiguration { readonly settingsSearchBuildId?: number; readonly settingsSearchUrl?: string; + readonly tasConfig?: { + endpoint: string; + telemetryEventName: string; + featuresTelemetryPropertyName: string; + assignmentContextTelemetryPropertyName: string; + }; + readonly experimentsUrl?: string; readonly extensionsGallery?: { diff --git a/src/vs/platform/telemetry/common/telemetry.ts b/src/vs/platform/telemetry/common/telemetry.ts index e0a9a30e55c..1acf9259110 100644 --- a/src/vs/platform/telemetry/common/telemetry.ts +++ b/src/vs/platform/telemetry/common/telemetry.ts @@ -46,6 +46,8 @@ export interface ITelemetryService { getTelemetryInfo(): Promise; + setExperimentProperty(name: string, value: string): void; + isOptedIn: boolean; } diff --git a/src/vs/platform/telemetry/common/telemetryService.ts b/src/vs/platform/telemetry/common/telemetryService.ts index 49c4d59d5d5..1e1c6fcb583 100644 --- a/src/vs/platform/telemetry/common/telemetryService.ts +++ b/src/vs/platform/telemetry/common/telemetryService.ts @@ -31,6 +31,7 @@ export class TelemetryService implements ITelemetryService { private _appender: ITelemetryAppender; private _commonProperties: Promise<{ [name: string]: any; }>; + private _experimentProperties: { [name: string]: string } = {}; private _piiPaths: string[]; private _userOptIn: boolean; private _enabled: boolean; @@ -79,6 +80,10 @@ export class TelemetryService implements ITelemetryService { } } + setExperimentProperty(name: string, value: string): void { + this._experimentProperties[name] = value; + } + setEnabled(value: boolean): void { this._enabled = value; } @@ -119,6 +124,9 @@ export class TelemetryService implements ITelemetryService { // (first) add common properties data = mixin(data, values); + // (next) add experiment properties + data = mixin(data, this._experimentProperties); + // (last) remove all PII from data data = cloneAndChange(data, value => { if (typeof value === 'string') { diff --git a/src/vs/platform/telemetry/common/telemetryUtils.ts b/src/vs/platform/telemetry/common/telemetryUtils.ts index ae0a2110009..392c3ad8ea8 100644 --- a/src/vs/platform/telemetry/common/telemetryUtils.ts +++ b/src/vs/platform/telemetry/common/telemetryUtils.ts @@ -28,6 +28,7 @@ export const NullTelemetryService = new class implements ITelemetryService { return this.publicLogError(eventName, data as ITelemetryData); } + setExperimentProperty() { } setEnabled() { } isOptedIn = true; getTelemetryInfo(): Promise { diff --git a/src/vs/workbench/services/experiment/common/experimentService.ts b/src/vs/workbench/services/experiment/common/experimentService.ts new file mode 100644 index 00000000000..758c525e263 --- /dev/null +++ b/src/vs/workbench/services/experiment/common/experimentService.ts @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; + +export const ITASExperimentService = createDecorator('TASExperimentService'); + +export interface ITASExperimentService { + readonly _serviceBrand: undefined; + getTreatment(name: string): Promise; +} diff --git a/src/vs/workbench/services/experiment/electron-browser/experimentService.ts b/src/vs/workbench/services/experiment/electron-browser/experimentService.ts new file mode 100644 index 00000000000..734c61a142b --- /dev/null +++ b/src/vs/workbench/services/experiment/electron-browser/experimentService.ts @@ -0,0 +1,209 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as platform from 'vs/base/common/platform'; +import { IKeyValueStorage, IExperimentationTelemetry, IExperimentationFilterProvider, ExperimentationService as TASClient } from 'tas-client'; +import { MementoObject, Memento } from 'vs/workbench/common/memento'; +import { IProductService } from 'vs/platform/product/common/productService'; +import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry'; +import { IStorageService, StorageScope } from 'vs/platform/storage/common/storage'; +import { ITelemetryData } from 'vs/base/common/actions'; +import { ITASExperimentService } from 'vs/workbench/services/experiment/common/experimentService'; +import { registerSingleton } from 'vs/platform/instantiation/common/extensions'; + +const storageKey = 'VSCode.ABExp.FeatureData'; +const refetchInterval = 1000 * 60 * 30; // By default it's set up to 30 minutes. + +class MementoKeyValueStorage implements IKeyValueStorage { + constructor(private mementoObj: MementoObject) { } + + async getValue(key: string, defaultValue?: T | undefined): Promise { + const value = await this.mementoObj[key]; + return value || defaultValue; + } + + setValue(key: string, value: T): void { + this.mementoObj[key] = value; + } +} + +class ExperimentServiceTelemetry implements IExperimentationTelemetry { + constructor(private telemetryService: ITelemetryService) { } + + // __GDPR__COMMON__ "VSCode.ABExp.Features" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" } + // __GDPR__COMMON__ "abexp.assignmentcontext" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" } + setSharedProperty(name: string, value: string): void { + this.telemetryService.setExperimentProperty(name, value); + } + + postEvent(eventName: string, props: Map): void { + const data: ITelemetryData = {}; + for (const [key, value] of props.entries()) { + data[key] = value; + } + + /* __GDPR__ + "query-expfeature" : { + "ABExp.queriedFeature": { "classification": "SystemMetaData", "purpose": "FeatureInsight" } + } + */ + this.telemetryService.publicLog(eventName, data); + } +} + +class ExperimentServiceFilterProvider implements IExperimentationFilterProvider { + constructor( + private version: string, + private appName: string, + private machineId: string, + private targetPopulation: TargetPopulation + ) { } + + getFilterValue(filter: string): string | null { + switch (filter) { + case Filters.ApplicationVersion: + return this.version; // productService.version + case Filters.Build: + return this.appName; // productService.nameLong + case Filters.ClientId: + return this.machineId; + case Filters.Language: + return platform.language; + case Filters.TargetPopulation: + return this.targetPopulation; + default: + return ''; + } + } + + getFilters(): Map { + let filters: Map = new Map(); + let filterValues = Object.values(Filters); + for (let value of filterValues) { + filters.set(value, this.getFilterValue(value)); + } + + return filters; + } +} + +/* +Based upon the official VSCode currently existing filters in the +ExP backend for the VSCode cluster. +https://experimentation.visualstudio.com/Analysis%20and%20Experimentation/_git/AnE.ExP.TAS.TachyonHost.Configuration?path=%2FConfigurations%2Fvscode%2Fvscode.json&version=GBmaster +"X-MSEdge-Market": "detection.market", +"X-FD-Corpnet": "detection.corpnet", +"X-VSCode–AppVersion": "appversion", +"X-VSCode-Build": "build", +"X-MSEdge-ClientId": "clientid", +"X-VSCode-TargetPopulation": "targetpopulation", +"X-VSCode-Language": "language" +*/ + +enum Filters { + /** + * The market in which the extension is distributed. + */ + Market = 'X-MSEdge-Market', + + /** + * The corporation network. + */ + CorpNet = 'X-FD-Corpnet', + + /** + * Version of the application which uses experimentation service. + */ + ApplicationVersion = 'X-VSCode-AppVersion', + + /** + * Insiders vs Stable. + */ + Build = 'X-VSCode-Build', + + /** + * Client Id which is used as primary unit for the experimentation. + */ + ClientId = 'X-MSEdge-ClientId', + + /** + * The language in use by VS Code + */ + Language = 'X-VSCode-Language', + + /** + * The target population. + * This is used to separate internal, early preview, GA, etc. + */ + TargetPopulation = 'X-VSCode-TargetPopulation', +} + +enum TargetPopulation { + Team = 'team', + Internal = 'internal', + Insiders = 'insider', + Public = 'public', +} + +export class ExperimentService implements ITASExperimentService { + _serviceBrand: undefined; + private tasClient: Promise | undefined; + private static MEMENTO_ID = 'experiment.service.memento'; + + constructor( + @IProductService private productService: IProductService, + @ITelemetryService private telemetryService: ITelemetryService, + @IStorageService private storageService: IStorageService + ) { + + if (this.productService.tasConfig) { + this.tasClient = this.setupTASClient(); + } + } + + async getTreatment(name: string): Promise { + if (!this.tasClient) { + return undefined; + } + + return (await this.tasClient).getTreatmentVariable('vscode', name); + } + + private async setupTASClient(): Promise { + const telemetryInfo = await this.telemetryService.getTelemetryInfo(); + const targetPopulation = telemetryInfo.msftInternal ? TargetPopulation.Internal : (this.productService.quality === 'stable' ? TargetPopulation.Public : TargetPopulation.Insiders); + const machineId = telemetryInfo.machineId; + const filterProvider = new ExperimentServiceFilterProvider( + this.productService.version, + this.productService.nameLong, + machineId, + targetPopulation + ); + + const memento = new Memento(ExperimentService.MEMENTO_ID, this.storageService); + const keyValueStorage = new MementoKeyValueStorage(memento.getMemento(StorageScope.GLOBAL)); + + const telemetry = new ExperimentServiceTelemetry(this.telemetryService); + + const tasConfig = this.productService.tasConfig!; + const tasClient = new TASClient({ + filterProviders: [filterProvider], + telemetry: telemetry, + storageKey: storageKey, + keyValueStorage: keyValueStorage, + featuresTelemetryPropertyName: tasConfig.featuresTelemetryPropertyName, + assignmentContextTelemetryPropertyName: tasConfig.assignmentContextTelemetryPropertyName, + telemetryEventName: tasConfig.telemetryEventName, + endpoint: tasConfig.endpoint, + refetchInterval: refetchInterval, + }); + + await tasClient.initializePromise; + return tasClient; + } +} + +registerSingleton(ITASExperimentService, ExperimentService, false); + diff --git a/src/vs/workbench/services/telemetry/browser/telemetryService.ts b/src/vs/workbench/services/telemetry/browser/telemetryService.ts index dd6ed7685bf..41b39345f1e 100644 --- a/src/vs/workbench/services/telemetry/browser/telemetryService.ts +++ b/src/vs/workbench/services/telemetry/browser/telemetryService.ts @@ -65,6 +65,10 @@ export class TelemetryService extends Disposable implements ITelemetryService { return this.impl.setEnabled(value); } + setExperimentProperty(name: string, value: string): void { + return this.impl.setExperimentProperty(name, value); + } + get isOptedIn(): boolean { return this.impl.isOptedIn; } diff --git a/src/vs/workbench/services/telemetry/electron-browser/telemetryService.ts b/src/vs/workbench/services/telemetry/electron-browser/telemetryService.ts index 2b32c4425df..99d36e3fa46 100644 --- a/src/vs/workbench/services/telemetry/electron-browser/telemetryService.ts +++ b/src/vs/workbench/services/telemetry/electron-browser/telemetryService.ts @@ -57,6 +57,10 @@ export class TelemetryService extends Disposable implements ITelemetryService { return this.impl.setEnabled(value); } + setExperimentProperty(name: string, value: string): void { + return this.impl.setExperimentProperty(name, value); + } + get isOptedIn(): boolean { return this.impl.isOptedIn; } diff --git a/src/vs/workbench/test/electron-browser/textsearch.perf.integrationTest.ts b/src/vs/workbench/test/electron-browser/textsearch.perf.integrationTest.ts index eab55d05b22..ed378fb9496 100644 --- a/src/vs/workbench/test/electron-browser/textsearch.perf.integrationTest.ts +++ b/src/vs/workbench/test/electron-browser/textsearch.perf.integrationTest.ts @@ -176,6 +176,9 @@ class TestTelemetryService implements ITelemetryService { public setEnabled(value: boolean): void { } + public setExperimentProperty(name: string, value: string): void { + } + public publicLog(eventName: string, data?: any): Promise { const event = { name: eventName, data: data }; this.events.push(event); diff --git a/src/vs/workbench/workbench.desktop.main.ts b/src/vs/workbench/workbench.desktop.main.ts index 1c13f30b47c..000a505240b 100644 --- a/src/vs/workbench/workbench.desktop.main.ts +++ b/src/vs/workbench/workbench.desktop.main.ts @@ -58,6 +58,7 @@ import 'vs/workbench/services/userDataSync/electron-browser/userDataSyncAccountS import 'vs/workbench/services/sharedProcess/electron-browser/sharedProcessService'; import 'vs/workbench/services/localizations/electron-browser/localizationsService'; import 'vs/workbench/services/path/electron-browser/pathService'; +import 'vs/workbench/services/experiment/electron-browser/experimentService'; import { registerSingleton } from 'vs/platform/instantiation/common/extensions'; import { ICredentialsService } from 'vs/platform/credentials/common/credentials'; diff --git a/yarn.lock b/yarn.lock index 4e23e2cd735..3843414ac08 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1108,6 +1108,13 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== +axios@^0.19.0: + version "0.19.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27" + integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA== + dependencies: + follow-redirects "1.5.10" + azure-storage@^2.10.2: version "2.10.2" resolved "https://registry.yarnpkg.com/azure-storage/-/azure-storage-2.10.2.tgz#3bcabdbf10e72fd0990db81116e49023c4a675b6" @@ -2371,7 +2378,7 @@ debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3: dependencies: ms "2.0.0" -debug@3.1.0: +debug@3.1.0, debug@=3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== @@ -3566,6 +3573,13 @@ flush-write-stream@^1.0.0, flush-write-stream@^1.0.2: inherits "^2.0.1" readable-stream "^2.0.4" +follow-redirects@1.5.10: + version "1.5.10" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" + integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== + dependencies: + debug "=3.1.0" + for-in@^0.1.5: version "0.1.5" resolved "https://registry.yarnpkg.com/for-in/-/for-in-0.1.5.tgz#007374e2b6d5c67420a1479bdb75a04872b738c4" @@ -9027,6 +9041,13 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.2" +tas-client@^0.0.950: + version "0.0.950" + resolved "https://registry.yarnpkg.com/tas-client/-/tas-client-0.0.950.tgz#0fadc684721d5bc6d6af03b09e1ff5a83a5186fc" + integrity sha512-AvCNjvfouxJyKln+TsobOBO5KmXklL9+FlxrEPlIgaixy1TxCC2v2Vs/MflCiyHlGl+BeIStP4oAVPqo5c0pIA== + dependencies: + axios "^0.19.0" + temp@^0.8.3: version "0.8.3" resolved "https://registry.yarnpkg.com/temp/-/temp-0.8.3.tgz#e0c6bc4d26b903124410e4fed81103014dfc1f59" -- GitLab