Add file walk telemetry (#9640)

src/vs/platform/search/common/search.ts

@@ -71,6 +71,14 @@ export interface ISearchProgressItem extends IFileMatch, IProgress {
 export interface ISearchComplete {
 	limitHit?: boolean;
 	results: IFileMatch[];
+	stats: ISearchStats;
+}
+
+export interface ISearchStats {
+	fileWalkStartTime: number;
+	fileWalkResultTime: number;
+	directoriesWalked: number;
+	filesWalked: number;
 }
 
 

src/vs/workbench/parts/search/browser/openAnythingHandler.ts

@@ -6,6 +6,7 @@
 'use strict';
 
 import * as arrays from 'vs/base/common/arrays';
+import * as objects from 'vs/base/common/objects';
 import {TPromise} from 'vs/base/common/winjs.base';
 import nls = require('vs/nls');
 import {ThrottledDelayer} from 'vs/base/common/async';
@@ -25,6 +26,7 @@ import * as openSymbolHandler from 'vs/workbench/parts/search/browser/openSymbol
 /* tslint:enable:no-unused-variable */
 import {IMessageService, Severity} from 'vs/platform/message/common/message';
 import {IInstantiationService} from 'vs/platform/instantiation/common/instantiation';
+import {ISearchStats} from 'vs/platform/search/common/search';
 import {ITelemetryService} from 'vs/platform/telemetry/common/telemetry';
 import {IWorkspaceContextService} from 'vs/workbench/services/workspace/common/contextService';
 import {IConfigurationService} from 'vs/platform/configuration/common/configuration';
@@ -40,11 +42,16 @@ interface ITimerEventData {
 	unsortedResultDuration: number;
 	sortedResultDuration: number;
 	numberOfResultEntries: number;
+	fileWalkStartDuration?: number;
+	fileWalkResultDuration?: number;
+	directoriesWalked?: number;
+	filesWalked?: number;
 }
 
 interface ITelemetryData {
 	fromCache: boolean;
 	searchLength: number;
+	searchStats?: ISearchStats;
 	unsortedResultTime: number;
 	sortedResultTime: number;
 	numberOfResultEntries: number;
@@ -133,6 +140,7 @@ export class OpenAnythingHandler extends QuickOpenHandler {
 		// The throttler needs a factory for its promises
 		let promiseFactory = () => {
 			let receivedFileResults = false;
+			let searchStats: ISearchStats;
 
 			// Symbol Results (unless a range is specified)
 			let resultPromises: TPromise<QuickOpenModel>[] = [];
@@ -164,8 +172,9 @@ export class OpenAnythingHandler extends QuickOpenHandler {
 			}
 
 			// File Results
-			resultPromises.push(this.openFileHandler.getResults(searchValue).then((results: QuickOpenModel) => {
+			resultPromises.push(this.openFileHandler.getResultsWithStats(searchValue).then(([results, stats]) => {
 				receivedFileResults = true;
+				searchStats = stats;
 
 				return results;
 			}));
@@ -204,6 +213,7 @@ export class OpenAnythingHandler extends QuickOpenHandler {
 				timerEvent.data = this.createTimerEventData(startTime, {
 					fromCache: false,
 					searchLength: searchValue.length,
+					searchStats: searchStats,
 					unsortedResultTime,
 					sortedResultTime,
 					numberOfResultEntries: result.length
@@ -372,12 +382,19 @@ export class OpenAnythingHandler extends QuickOpenHandler {
 	}
 
 	private createTimerEventData(startTime: number, telemetry: ITelemetryData): ITimerEventData {
-		return {
+		const data: ITimerEventData = {
 			fromCache: telemetry.fromCache,
 			searchLength: telemetry.searchLength,
 			unsortedResultDuration: telemetry.unsortedResultTime - startTime,
 			sortedResultDuration: telemetry.sortedResultTime - startTime,
 			numberOfResultEntries: telemetry.numberOfResultEntries
 		};
+		const stats = telemetry.searchStats;
+		return stats ? objects.assign(data, {
+			fileWalkStartDuration: stats.fileWalkStartTime - startTime,
+			fileWalkResultDuration: stats.fileWalkResultTime - startTime,
+			directoriesWalked: stats.directoriesWalked,
+			filesWalked: stats.filesWalked
+		}) : data;
 	}
 }
\ No newline at end of file

src/vs/workbench/parts/search/browser/openFileHandler.ts

@@ -20,7 +20,7 @@ import {IResourceInput} from 'vs/platform/editor/common/editor';
 import {IWorkbenchEditorService} from 'vs/workbench/services/editor/common/editorService';
 import {IConfigurationService} from 'vs/platform/configuration/common/configuration';
 import {IInstantiationService} from 'vs/platform/instantiation/common/instantiation';
-import {IQueryOptions, ISearchService} from 'vs/platform/search/common/search';
+import {IQueryOptions, ISearchService, ISearchStats} from 'vs/platform/search/common/search';
 import {IWorkspaceContextService} from 'vs/platform/workspace/common/workspace';
 
 export class FileEntry extends EditorQuickOpenEntry {
@@ -100,20 +100,25 @@ export class OpenFileHandler extends QuickOpenHandler {
 	}
 
 	public getResults(searchValue: string): TPromise<QuickOpenModel> {
+		return this.getResultsWithStats(searchValue)
+			.then(result => result[0]);
+	}
+
+	public getResultsWithStats(searchValue: string): TPromise<[QuickOpenModel, ISearchStats]> {
 		searchValue = searchValue.trim();
-		let promise: TPromise<QuickOpenEntry[]>;
+		let promise: TPromise<[QuickOpenEntry[], ISearchStats]>;
 
 		// Respond directly to empty search
 		if (!searchValue) {
-			promise = TPromise.as([]);
+			promise = TPromise.as(<[QuickOpenEntry[], ISearchStats]>[[], undefined]);
 		} else {
 			promise = this.doFindResults(searchValue);
 		}
 
-		return promise.then(e => new QuickOpenModel(e));
+		return promise.then(result => [new QuickOpenModel(result[0]), result[1]]);
 	}
 
-	private doFindResults(searchValue: string): TPromise<QuickOpenEntry[]> {
+	private doFindResults(searchValue: string): TPromise<[QuickOpenEntry[], ISearchStats]> {
 		const query: IQueryOptions = {
 			folderResources: this.contextService.getWorkspace() ? [this.contextService.getWorkspace().resource] : [],
 			extraFileResources: getOutOfWorkspaceEditorResources(this.editorGroupService, this.contextService),
@@ -131,7 +136,7 @@ export class OpenFileHandler extends QuickOpenHandler {
 				results.push(this.instantiationService.createInstance(FileEntry, label, description, fileMatch.resource));
 			}
 
-			return results;
+			return [results, complete.stats];
 		});
 	}
 

src/vs/workbench/parts/search/test/common/searchModel.test.ts

@@ -14,7 +14,7 @@ import { SearchModel } from 'vs/workbench/parts/search/common/searchModel';
 import URI from 'vs/base/common/uri';
 import {IFileMatch, ILineMatch} from 'vs/platform/search/common/search';
 import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
-import { ISearchService, ISearchComplete, ISearchProgressItem } from 'vs/platform/search/common/search';
+import { ISearchService, ISearchComplete, ISearchProgressItem, ISearchStats } from 'vs/platform/search/common/search';
 import { Range } from 'vs/editor/common/core/range';
 import { createMockModelService } from 'vs/test/utils/servicesTestUtils';
 import { IModelService } from 'vs/editor/common/services/modelService';
@@ -24,6 +24,13 @@ suite('SearchModel', () => {
 	let instantiationService: TestInstantiationService;
 	let restoreStubs;
 
+	const testSearchStats: ISearchStats = {
+		fileWalkStartTime: 0,
+		fileWalkResultTime: 1,
+		directoriesWalked: 2,
+		filesWalked: 3
+	};
+
 	setup(() => {
 		restoreStubs= [];
 		instantiationService= new TestInstantiationService();
@@ -72,7 +79,7 @@ suite('SearchModel', () => {
 
 		promise.progress(results[0]);
 		promise.progress(results[1]);
-		promise.complete({results: []});
+		promise.complete({results: [], stats: testSearchStats});
 
 		result.done(() => {
 			let actual= testObject.searchResult.matches();
@@ -138,7 +145,7 @@ suite('SearchModel', () => {
 		let result= testObject.search({contentPattern: {pattern: 'somestring'}, type: 1});
 
 		promise.progress(aRawMatch('file://c:/1', aLineMatch('some preview')));
-		promise.complete({results: []});
+		promise.complete({results: [], stats: testSearchStats});
 
 		result.done(() => {
 			assert.ok(target1.calledTwice);

src/vs/workbench/services/search/node/fileSearch.ts

@@ -13,11 +13,11 @@ import arrays = require('vs/base/common/arrays');
 import strings = require('vs/base/common/strings');
 import types = require('vs/base/common/types');
 import glob = require('vs/base/common/glob');
-import {IProgress} from 'vs/platform/search/common/search';
+import {IProgress, ISearchStats} from 'vs/platform/search/common/search';
 
 import extfs = require('vs/base/node/extfs');
 import flow = require('vs/base/node/flow');
-import {ISerializedFileMatch, IRawSearch, ISearchEngine} from './search';
+import {ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine} from './search';
 
 export class FileWalker {
 	private config: IRawSearch;
@@ -30,6 +30,9 @@ export class FileWalker {
 	private isLimitHit: boolean;
 	private resultCount: number;
 	private isCanceled: boolean;
+	private fileWalkStartTime: number;
+	private directoriesWalked: number;
+	private filesWalked: number;
 
 	private walkedPaths: { [path: string]: boolean; };
 
@@ -43,6 +46,8 @@ export class FileWalker {
 		this.walkedPaths = Object.create(null);
 		this.resultCount = 0;
 		this.isLimitHit = false;
+		this.directoriesWalked = 0;
+		this.filesWalked = 0;
 
 		if (this.filePattern) {
 			this.filePattern = this.filePattern.replace(/\\/g, '/'); // Normalize file patterns to forward slashes
@@ -55,6 +60,7 @@ export class FileWalker {
 	}
 
 	public walk(rootFolders: string[], extraFiles: string[], onResult: (result: ISerializedFileMatch, size: number) => void, done: (error: Error, isLimitHit: boolean) => void): void {
+		this.fileWalkStartTime = Date.now();
 
 		// Support that the file pattern is a full path to a file that exists
 		this.checkFilePatternAbsoluteMatch((exists, size) => {
@@ -86,6 +92,7 @@ export class FileWalker {
 
 			// For each root folder
 			flow.parallel(rootFolders, (absolutePath, perEntryCallback) => {
+				this.directoriesWalked++;
 				extfs.readdir(absolutePath, (error: Error, files: string[]) => {
 					if (error || this.isCanceled || this.isLimitHit) {
 						return perEntryCallback(null, null);
@@ -111,6 +118,15 @@ export class FileWalker {
 		});
 	}
 
+	public getStats(): ISearchStats {
+		return {
+			fileWalkStartTime: this.fileWalkStartTime,
+			fileWalkResultTime: Date.now(),
+			directoriesWalked: this.directoriesWalked,
+			filesWalked: this.filesWalked
+		};
+	}
+
 	private checkFilePatternAbsoluteMatch(clb: (exists: boolean, size?: number) => void): void {
 		if (!this.filePattern || !paths.isAbsolute(this.filePattern)) {
 			return clb(false);
@@ -174,6 +190,7 @@ export class FileWalker {
 
 					// Directory: Follow directories
 					if (stat.isDirectory()) {
+						this.directoriesWalked++;
 
 						// to really prevent loops with links we need to resolve the real path of them
 						return this.realPathIfNeeded(currentAbsolutePath, lstat, (error, realpath) => {
@@ -200,6 +217,7 @@ export class FileWalker {
 
 					// File: Check for match on file pattern and include pattern
 					else {
+						this.filesWalked++;
 						if (currentRelativePathWithSlashes === this.filePattern) {
 							return clb(null); // ignore file if its path matches with the file pattern because checkFilePatternRelativeMatch() takes care of those
 						}
@@ -290,8 +308,13 @@ export class Engine implements ISearchEngine {
 		this.walker = new FileWalker(config);
 	}
 
-	public search(onResult: (result: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, isLimitHit: boolean) => void): void {
-		this.walker.walk(this.rootFolders, this.extraFiles, onResult, done);
+	public search(onResult: (result: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
+		this.walker.walk(this.rootFolders, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => {
+			done(err, {
+				limitHit: isLimitHit,
+				stats: this.walker.getStats()
+			});
+		});
 	}
 
 	public cancel(): void {

src/vs/workbench/services/search/node/rawSearchService.ts

@@ -56,16 +56,14 @@ export class SearchService implements IRawSearchService {
 				}
 			}, (progress) => {
 				p(progress);
-			}, (error, isLimitHit) => {
+			}, (error, stats) => {
 				if (batch.length) {
 					p(batch);
 				}
 				if (error) {
 					e(error);
 				} else {
-					c({
-						limitHit: isLimitHit
-					});
+					c(stats);
 				}
 			});
 		}, () => engine.cancel());

src/vs/workbench/services/search/node/search.ts

@@ -7,7 +7,7 @@
 
 import { PPromise } from 'vs/base/common/winjs.base';
 import glob = require('vs/base/common/glob');
-import { IProgress, ILineMatch, IPatternInfo } from 'vs/platform/search/common/search';
+import { IProgress, ILineMatch, IPatternInfo, ISearchStats } from 'vs/platform/search/common/search';
 
 export interface IRawSearch {
 	rootFolders: string[];
@@ -27,12 +27,13 @@ export interface IRawSearchService {
 }
 
 export interface ISearchEngine {
-	search: (onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, isLimitHit: boolean) => void) => void;
+	search: (onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void) => void;
 	cancel: () => void;
 }
 
 export interface ISerializedSearchComplete {
 	limitHit: boolean;
+	stats: ISearchStats;
 }
 
 export interface ISerializedFileMatch {

src/vs/workbench/services/search/node/searchService.ts

@@ -74,7 +74,11 @@ export class SearchService implements ISearchService {
 				// on Complete
 				(complete) => {
 					flushLocalResultsOnce();
-					onComplete({ results: complete.results.filter((match) => typeof localResults[match.resource.toString()] === 'undefined'), limitHit: complete.limitHit }); // dont override local results
+					onComplete({
+						limitHit: complete.limitHit,
+						results: complete.results.filter((match) => typeof localResults[match.resource.toString()] === 'undefined'), // dont override local results
+						stats: complete.stats
+					});
 				},
 
 				// on Error
@@ -242,7 +246,8 @@ export class DiskSearch {
 			request.done((complete) => {
 				c({
 					limitHit: complete.limitHit,
-					results: result
+					results: result,
+					stats: complete.stats
 				});
 			}, e, (data) => {
 

src/vs/workbench/services/search/node/textSearch.ts

@@ -14,7 +14,7 @@ import {ILineMatch, IProgress} from 'vs/platform/search/common/search';
 import {detectMimeAndEncodingFromBuffer} from 'vs/base/node/mime';
 import {FileWalker} from 'vs/workbench/services/search/node/fileSearch';
 import {UTF16le, UTF16be, UTF8, UTF8_with_bom, encodingExists, decode} from 'vs/base/node/encoding';
-import {ISerializedFileMatch, IRawSearch, ISearchEngine} from './search';
+import {ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine} from './search';
 
 interface ReadLinesOptions {
 	bufferLength: number;
@@ -59,7 +59,7 @@ export class Engine implements ISearchEngine {
 		this.walker.cancel();
 	}
 
-	public search(onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, isLimitHit: boolean) => void): void {
+	public search(onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
 		let resultCounter = 0;
 
 		let progress = () => {
@@ -80,7 +80,10 @@ export class Engine implements ISearchEngine {
 			// Emit done()
 			if (this.worked === this.total && this.walkerIsDone && !this.isDone) {
 				this.isDone = true;
-				done(this.walkerError, this.limitReached);
+				done(this.walkerError, {
+					limitHit: this.limitReached,
+					stats: this.walker.getStats()
+				});
 			}
 		};
 

src/vs/workbench/services/search/test/node/searchService.test.ts

@@ -8,7 +8,7 @@
 import assert = require('assert');
 
 import {IProgress} from 'vs/platform/search/common/search';
-import {ISearchEngine, ISerializedFileMatch} from 'vs/workbench/services/search/node/search';
+import {ISearchEngine, ISerializedFileMatch, ISerializedSearchComplete} from 'vs/workbench/services/search/node/search';
 import {SearchService as RawSearchService} from 'vs/workbench/services/search/node/rawSearchService';
 import {DiskSearch} from 'vs/workbench/services/search/node/searchService';
 
@@ -18,13 +18,21 @@ class TestSearchEngine implements ISearchEngine {
 	constructor(private result: () => ISerializedFileMatch) {
 	}
 
-	public search(onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, isLimitHit: boolean) => void): void {
+	public search(onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
 		const self = this;
 		(function next() {
 			process.nextTick(() => {
 				const result = self.result();
 				if (!result) {
-					done(null, false);
+					done(null, {
+						limitHit: false,
+						stats: {
+							fileWalkStartTime: 0,
+							fileWalkResultTime: 1,
+							directoriesWalked: 2,
+							filesWalked: 3
+						}
+					});
 				} else {
 					onResult(result);
 					next();