From 0f776fe45882c42fdeca98139676e94a2afa8fee Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Thu, 4 Aug 2016 16:58:36 -0700 Subject: [PATCH] #55 Sort and limit file results in search process --- src/vs/base/common/comparers.ts | 63 ++++- .../parts/quickopen/browser/quickOpenModel.ts | 68 +---- src/vs/platform/search/common/search.ts | 18 +- .../search/browser/openAnythingHandler.ts | 150 ++++++----- .../parts/search/browser/openFileHandler.ts | 17 +- .../parts/search/common/searchQuery.ts | 2 + .../search/test/common/searchModel.test.ts | 6 +- .../services/search/node/fileSearch.ts | 40 ++- .../services/search/node/rawSearchService.ts | 241 +++++++++++++++++- .../workbench/services/search/node/search.ts | 15 +- .../services/search/node/searchIpc.ts | 6 + .../services/search/node/searchService.ts | 14 +- .../services/search/node/textSearch.ts | 10 +- .../services/search/test/node/search.test.ts | 25 +- .../search/test/node/searchService.test.ts | 199 +++++++++++++-- .../parts/quickOpen/quickopen.perf.test.ts | 28 +- 16 files changed, 700 insertions(+), 202 deletions(-) diff --git a/src/vs/base/common/comparers.ts b/src/vs/base/common/comparers.ts index 702add75f27..7d29ba9a8b0 100644 --- a/src/vs/base/common/comparers.ts +++ b/src/vs/base/common/comparers.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; +import scorer = require('vs/base/common/scorer'); import strings = require('vs/base/common/strings'); const FileNameMatch = /^(.*)\.([^.]*)|([^.]+)$/; @@ -75,4 +76,64 @@ export function compareByPrefix(one: string, other: string, lookFor: string): nu } return 0; -} \ No newline at end of file +} + +export interface IScorableResourceAccessor { + getLabel(T): string; + getResourcePath(T): string; +} + +export function compareByScore(elementA: T, elementB: T, accessor: IScorableResourceAccessor, lookFor: string, lookForNormalizedLower: string, scorerCache?: { [key: string]: number }): number { + const labelA = accessor.getLabel(elementA); + const labelB = accessor.getLabel(elementB); + + // treat prefix matches highest in any case + const prefixCompare = compareByPrefix(labelA, labelB, lookFor); + if (prefixCompare) { + return prefixCompare; + } + + // Give higher importance to label score + const labelAScore = scorer.score(labelA, lookFor, scorerCache); + const labelBScore = scorer.score(labelB, lookFor, scorerCache); + + // Useful for understanding the scoring + // elementA.setPrefix(labelAScore + ' '); + // elementB.setPrefix(labelBScore + ' '); + + if (labelAScore !== labelBScore) { + return labelAScore > labelBScore ? -1 : 1; + } + + // Score on full resource path comes next (if available) + let resourcePathA = accessor.getResourcePath(elementA); + let resourcePathB = accessor.getResourcePath(elementB); + if (resourcePathA && resourcePathB) { + const resourceAScore = scorer.score(resourcePathA, lookFor, scorerCache); + const resourceBScore = scorer.score(resourcePathB, lookFor, scorerCache); + + // Useful for understanding the scoring + // elementA.setPrefix(elementA.getPrefix() + ' ' + resourceAScore + ': '); + // elementB.setPrefix(elementB.getPrefix() + ' ' + resourceBScore + ': '); + + if (resourceAScore !== resourceBScore) { + return resourceAScore > resourceBScore ? -1 : 1; + } + } + + // At this place, the scores are identical so we check for string lengths and favor shorter ones + if (labelA.length !== labelB.length) { + return labelA.length < labelB.length ? -1 : 1; + } + + if (resourcePathA && resourcePathB && resourcePathA.length !== resourcePathB.length) { + return resourcePathA.length < resourcePathB.length ? -1 : 1; + } + + // Finally compare by label or resource path + if (labelA === labelB && resourcePathA && resourcePathB) { + return compareAnything(resourcePathA, resourcePathB, lookForNormalizedLower); + } + + return compareAnything(labelA, labelB, lookForNormalizedLower); +} diff --git a/src/vs/base/parts/quickopen/browser/quickOpenModel.ts b/src/vs/base/parts/quickopen/browser/quickOpenModel.ts index 8e7dbd5e4dc..ad55b7551fb 100644 --- a/src/vs/base/parts/quickopen/browser/quickOpenModel.ts +++ b/src/vs/base/parts/quickopen/browser/quickOpenModel.ts @@ -16,12 +16,11 @@ import paths = require('vs/base/common/paths'); import {IQuickNavigateConfiguration, IModel, IDataSource, IFilter, IAccessiblityProvider, IRenderer, IRunner, Mode} from 'vs/base/parts/quickopen/common/quickOpen'; import {IActionProvider} from 'vs/base/parts/tree/browser/actionsRenderer'; import {Action, IAction, IActionRunner} from 'vs/base/common/actions'; -import {compareAnything, compareByPrefix} from 'vs/base/common/comparers'; +import {compareAnything, compareByScore as doCompareByScore} from 'vs/base/common/comparers'; import {ActionBar, IActionItem} from 'vs/base/browser/ui/actionbar/actionbar'; import {LegacyRenderer, ILegacyTemplateData} from 'vs/base/parts/tree/browser/treeDefaults'; import {HighlightedLabel} from 'vs/base/browser/ui/highlightedlabel/highlightedLabel'; import DOM = require('vs/base/browser/dom'); -import scorer = require('vs/base/common/scorer'); export interface IContext { event: any; @@ -35,6 +34,18 @@ export interface IHighlight { let IDS = 0; +class EntryAccessor { + + public static getLabel(entry: QuickOpenEntry) { + return entry.getLabel(); + } + + public static getResourcePath(entry: QuickOpenEntry) { + const resource = entry.getResource(); + return resource && resource.fsPath; + } +} + export class QuickOpenEntry { private id: string; private labelHighlights: IHighlight[]; @@ -183,58 +194,7 @@ export class QuickOpenEntry { } public static compareByScore(elementA: QuickOpenEntry, elementB: QuickOpenEntry, lookFor: string, lookForNormalizedLower: string, scorerCache?: { [key: string]: number }): number { - const labelA = elementA.getLabel(); - const labelB = elementB.getLabel(); - - // treat prefix matches highest in any case - const prefixCompare = compareByPrefix(labelA, labelB, lookFor); - if (prefixCompare) { - return prefixCompare; - } - - // Give higher importance to label score - const labelAScore = scorer.score(labelA, lookFor, scorerCache); - const labelBScore = scorer.score(labelB, lookFor, scorerCache); - - // Useful for understanding the scoring - // elementA.setPrefix(labelAScore + ' '); - // elementB.setPrefix(labelBScore + ' '); - - if (labelAScore !== labelBScore) { - return labelAScore > labelBScore ? -1 : 1; - } - - // Score on full resource path comes next (if available) - let resourceA = elementA.getResource(); - let resourceB = elementB.getResource(); - if (resourceA && resourceB) { - const resourceAScore = scorer.score(resourceA.fsPath, lookFor, scorerCache); - const resourceBScore = scorer.score(resourceB.fsPath, lookFor, scorerCache); - - // Useful for understanding the scoring - // elementA.setPrefix(elementA.getPrefix() + ' ' + resourceAScore + ': '); - // elementB.setPrefix(elementB.getPrefix() + ' ' + resourceBScore + ': '); - - if (resourceAScore !== resourceBScore) { - return resourceAScore > resourceBScore ? -1 : 1; - } - } - - // At this place, the scores are identical so we check for string lengths and favor shorter ones - if (labelA.length !== labelB.length) { - return labelA.length < labelB.length ? -1 : 1; - } - - if (resourceA && resourceB && resourceA.fsPath.length !== resourceB.fsPath.length) { - return resourceA.fsPath.length < resourceB.fsPath.length ? -1 : 1; - } - - // Finally compare by label or resource path - if (labelA === labelB && resourceA && resourceB) { - return compareAnything(resourceA.fsPath, resourceB.fsPath, lookForNormalizedLower); - } - - return compareAnything(labelA, labelB, lookForNormalizedLower); + return doCompareByScore(elementA, elementB, EntryAccessor, lookFor, lookForNormalizedLower, scorerCache); } /** diff --git a/src/vs/platform/search/common/search.ts b/src/vs/platform/search/common/search.ts index 4459476c572..ffd00494f5b 100644 --- a/src/vs/platform/search/common/search.ts +++ b/src/vs/platform/search/common/search.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {PPromise} from 'vs/base/common/winjs.base'; +import {PPromise, TPromise} from 'vs/base/common/winjs.base'; import uri from 'vs/base/common/uri'; import glob = require('vs/base/common/glob'); import {IFilesConfiguration} from 'vs/platform/files/common/files'; @@ -19,6 +19,7 @@ export const ISearchService = createDecorator(ID); export interface ISearchService { _serviceBrand: any; search(query: ISearchQuery): PPromise; + clearCache(cacheKey: string): TPromise; } export interface IQueryOptions { @@ -28,6 +29,8 @@ export interface IQueryOptions { excludePattern?: glob.IExpression; includePattern?: glob.IExpression; maxResults?: number; + sortByScore?: boolean; + cacheKey?: string; fileEncoding?: string; } @@ -75,6 +78,19 @@ export interface ISearchComplete { } export interface ISearchStats { + fromCache: boolean; + resultCount: number; + unsortedResultTime?: number; + sortedResultTime?: number; +} + +export interface ICachedSearchStats extends ISearchStats { + cacheLookupStartTime: number; + cacheLookupResultTime: number; + cacheEntryCount: number; +} + +export interface IUncachedSearchStats extends ISearchStats { fileWalkStartTime: number; fileWalkResultTime: number; directoriesWalked: number; diff --git a/src/vs/workbench/parts/search/browser/openAnythingHandler.ts b/src/vs/workbench/parts/search/browser/openAnythingHandler.ts index 96c2ee93a74..cdbd0d28417 100644 --- a/src/vs/workbench/parts/search/browser/openAnythingHandler.ts +++ b/src/vs/workbench/parts/search/browser/openAnythingHandler.ts @@ -16,6 +16,7 @@ import scorer = require('vs/base/common/scorer'); import paths = require('vs/base/common/paths'); import labels = require('vs/base/common/labels'); import strings = require('vs/base/common/strings'); +import uuid = require('vs/base/common/uuid'); import {IRange} from 'vs/editor/common/editorCommon'; import {IAutoFocus} from 'vs/base/parts/quickopen/common/quickOpen'; import {QuickOpenEntry, QuickOpenModel} from 'vs/base/parts/quickopen/browser/quickOpenModel'; @@ -26,35 +27,52 @@ import * as openSymbolHandler from 'vs/workbench/parts/search/browser/openSymbol /* tslint:enable:no-unused-variable */ import {IMessageService, Severity} from 'vs/platform/message/common/message'; import {IInstantiationService} from 'vs/platform/instantiation/common/instantiation'; -import {ISearchStats} from 'vs/platform/search/common/search'; +import {ISearchStats, ICachedSearchStats, IUncachedSearchStats} from 'vs/platform/search/common/search'; import {ITelemetryService} from 'vs/platform/telemetry/common/telemetry'; import {IWorkspaceContextService} from 'vs/workbench/services/workspace/common/contextService'; import {IConfigurationService} from 'vs/platform/configuration/common/configuration'; +const objects_assign: (destination: T, source: U) => T & U = objects.assign; + interface ISearchWithRange { search: string; range: IRange; } interface ITimerEventData { - fromCache: boolean; searchLength: number; unsortedResultDuration: number; sortedResultDuration: number; - numberOfResultEntries: number; - fileWalkStartDuration?: number; - fileWalkResultDuration?: number; - directoriesWalked?: number; - filesWalked?: number; + resultCount: number; + symbols: { + fromCache: boolean; + }; + files: { + fromCache: boolean; + unsortedResultDuration: number; + sortedResultDuration: number; + resultCount: number; + } & ({ + fileWalkStartDuration: number; + fileWalkResultDuration: number; + directoriesWalked: number; + filesWalked: number; + } | { + cacheLookupStartDuration: number; + cacheLookupResultDuration: number; + cacheEntryCount: number; + }); } interface ITelemetryData { - fromCache: boolean; searchLength: number; - searchStats?: ISearchStats; unsortedResultTime: number; sortedResultTime: number; - numberOfResultEntries: number; + resultCount: number; + symbols: { + fromCache: boolean; + }; + files: ISearchStats; } // OpenSymbolHandler is used from an extension and must be in the main bundle file so it can load @@ -71,11 +89,12 @@ export class OpenAnythingHandler extends QuickOpenHandler { private openSymbolHandler: OpenSymbolHandler; private openFileHandler: OpenFileHandler; - private resultsToSearchCache: { [searchValue: string]: QuickOpenEntry[]; }; + private symbolResultsToSearchCache: { [searchValue: string]: QuickOpenEntry[]; }; private delayer: ThrottledDelayer; private pendingSearch: TPromise; private isClosed: boolean; private scorerCache: { [key: string]: number }; + private cacheKey: string; constructor( @IMessageService private messageService: IMessageService, @@ -96,8 +115,9 @@ export class OpenAnythingHandler extends QuickOpenHandler { skipSorting: true // we sort combined with file results }); - this.resultsToSearchCache = Object.create(null); + this.symbolResultsToSearchCache = Object.create(null); this.scorerCache = Object.create(null); + this.cacheKey = uuid.generateUuid(); this.delayer = new ThrottledDelayer(OpenAnythingHandler.SEARCH_DELAY); } @@ -129,13 +149,7 @@ export class OpenAnythingHandler extends QuickOpenHandler { } // Check Cache first - let cachedResults = this.getResultsFromCache(searchValue, searchWithRange ? searchWithRange.range : null); - if (cachedResults) { - const [viewResults, telemetry] = cachedResults; - timerEvent.data = this.createTimerEventData(startTime, telemetry); - timerEvent.stop(); - return TPromise.as(new QuickOpenModel(viewResults)); - } + let cachedSymbolResults = this.getSymbolResultsFromCache(searchValue, !!searchWithRange); // The throttler needs a factory for its promises let promiseFactory = () => { @@ -144,7 +158,9 @@ export class OpenAnythingHandler extends QuickOpenHandler { // Symbol Results (unless a range is specified) let resultPromises: TPromise[] = []; - if (!searchWithRange) { + if (cachedSymbolResults) { + resultPromises.push(TPromise.as(new QuickOpenModel(cachedSymbolResults))); + } else if (!searchWithRange) { let symbolSearchTimeoutPromiseFn: (timeout: number) => TPromise = (timeout) => { return TPromise.timeout(timeout).then(() => { @@ -172,7 +188,7 @@ export class OpenAnythingHandler extends QuickOpenHandler { } // File Results - resultPromises.push(this.openFileHandler.getResultsWithStats(searchValue).then(([results, stats]) => { + resultPromises.push(this.openFileHandler.getResultsWithStats(searchValue, this.cacheKey, OpenAnythingHandler.MAX_DISPLAYED_RESULTS).then(([results, stats]) => { receivedFileResults = true; searchStats = stats; @@ -190,10 +206,11 @@ export class OpenAnythingHandler extends QuickOpenHandler { } // Combine symbol results and file results - let result = [...results[0].entries, ...results[1].entries]; + const symbolResults = results[0].entries; + let result = [...symbolResults, ...results[1].entries]; - // Cache for fast lookup - this.resultsToSearchCache[searchValue] = result; + // // Cache for fast lookup + this.symbolResultsToSearchCache[searchValue] = symbolResults; // Sort const normalizedSearchValue = strings.stripWildcards(searchValue).toLowerCase(); @@ -211,12 +228,14 @@ export class OpenAnythingHandler extends QuickOpenHandler { }); timerEvent.data = this.createTimerEventData(startTime, { - fromCache: false, searchLength: searchValue.length, - searchStats: searchStats, unsortedResultTime, sortedResultTime, - numberOfResultEntries: result.length + resultCount: result.length, + symbols: { + fromCache: !!cachedSymbolResults + }, + files: searchStats }); timerEvent.stop(); return TPromise.as(new QuickOpenModel(viewResults)); @@ -228,8 +247,8 @@ export class OpenAnythingHandler extends QuickOpenHandler { return this.pendingSearch; }; - // Trigger through delayer to prevent accumulation while the user is typing - return this.delayer.trigger(promiseFactory); + // Trigger through delayer to prevent accumulation while the user is typing (except when expecting results to come from cache) + return cachedSymbolResults ? promiseFactory() : this.delayer.trigger(promiseFactory); } private extractRange(value: string): ISearchWithRange { @@ -280,14 +299,14 @@ export class OpenAnythingHandler extends QuickOpenHandler { return null; } - private getResultsFromCache(searchValue: string, range: IRange = null): [QuickOpenEntry[], ITelemetryData] { + private getSymbolResultsFromCache(searchValue: string, hasRange: boolean): QuickOpenEntry[] { if (paths.isAbsolute(searchValue)) { return null; // bypass cache if user looks up an absolute path where matching goes directly on disk } // Find cache entries by prefix of search value let cachedEntries: QuickOpenEntry[]; - for (let previousSearch in this.resultsToSearchCache) { + for (let previousSearch in this.symbolResultsToSearchCache) { // If we narrow down, we might be able to reuse the cached results if (searchValue.indexOf(previousSearch) === 0) { @@ -295,7 +314,7 @@ export class OpenAnythingHandler extends QuickOpenHandler { continue; // since a path character widens the search for potential more matches, require it in previous search too } - cachedEntries = this.resultsToSearchCache[previousSearch]; + cachedEntries = this.symbolResultsToSearchCache[previousSearch]; break; } } @@ -304,17 +323,16 @@ export class OpenAnythingHandler extends QuickOpenHandler { return null; } + if (hasRange) { + return []; + } + // Pattern match on results and adjust highlights let results: QuickOpenEntry[] = []; const normalizedSearchValueLowercase = strings.stripWildcards(searchValue).toLowerCase(); for (let i = 0; i < cachedEntries.length; i++) { let entry = cachedEntries[i]; - // Check for file entries if range is used - if (range && !(entry instanceof FileEntry)) { - continue; - } - // Check if this entry is a match for the search value const resource = entry.getResource(); // can be null for symbol results! let targetToMatch = resource ? labels.getPathLabel(resource, this.contextService) : entry.getLabel(); @@ -324,29 +342,8 @@ export class OpenAnythingHandler extends QuickOpenHandler { results.push(entry); } - const unsortedResultTime = Date.now(); - - // Sort - const compare = (elementA, elementB) => QuickOpenEntry.compareByScore(elementA, elementB, searchValue, normalizedSearchValueLowercase, this.scorerCache); - const viewResults = arrays.top(results, compare, OpenAnythingHandler.MAX_DISPLAYED_RESULTS); - const sortedResultTime = Date.now(); - - // Apply range and highlights - viewResults.forEach(entry => { - if (entry instanceof FileEntry) { - entry.setRange(range); - } - const {labelHighlights, descriptionHighlights} = QuickOpenEntry.highlight(entry, searchValue, true /* fuzzy highlight */); - entry.setHighlights(labelHighlights, descriptionHighlights); - }); - return [viewResults, { - fromCache: true, - searchLength: searchValue.length, - unsortedResultTime, - sortedResultTime, - numberOfResultEntries: results.length - }]; + return results; } public getGroupLabel(): string { @@ -366,8 +363,10 @@ export class OpenAnythingHandler extends QuickOpenHandler { this.cancelPendingSearch(); // Clear Cache - this.resultsToSearchCache = Object.create(null); + this.symbolResultsToSearchCache = Object.create(null); this.scorerCache = Object.create(null); + this.openFileHandler.clearCache(this.cacheKey); + this.cacheKey = uuid.generateUuid(); // Propagate this.openSymbolHandler.onClose(canceled); @@ -382,19 +381,30 @@ export class OpenAnythingHandler extends QuickOpenHandler { } private createTimerEventData(startTime: number, telemetry: ITelemetryData): ITimerEventData { - const data: ITimerEventData = { - fromCache: telemetry.fromCache, + const stats = telemetry.files; + const cached = stats as ICachedSearchStats; + const uncached = stats as IUncachedSearchStats; + return { searchLength: telemetry.searchLength, unsortedResultDuration: telemetry.unsortedResultTime - startTime, sortedResultDuration: telemetry.sortedResultTime - startTime, - numberOfResultEntries: telemetry.numberOfResultEntries + resultCount: telemetry.resultCount, + symbols: telemetry.symbols, + files: objects_assign({ + fromCache: stats.fromCache, + unsortedResultDuration: stats.unsortedResultTime - startTime, + sortedResultDuration: stats.sortedResultTime - startTime, + resultCount: stats.resultCount + }, stats.fromCache ? { + cacheLookupStartDuration: cached.cacheLookupStartTime - startTime, + cacheLookupResultDuration: cached.cacheLookupResultTime - startTime, + cacheEntryCount: cached.cacheEntryCount + } : { + fileWalkStartDuration: uncached.fileWalkStartTime - startTime, + fileWalkResultDuration: uncached.fileWalkResultTime - startTime, + directoriesWalked: uncached.directoriesWalked, + filesWalked: uncached.filesWalked + }) }; - const stats = telemetry.searchStats; - return stats ? objects.assign(data, { - fileWalkStartDuration: stats.fileWalkStartTime - startTime, - fileWalkResultDuration: stats.fileWalkResultTime - startTime, - directoriesWalked: stats.directoriesWalked, - filesWalked: stats.filesWalked - }) : data; } } \ No newline at end of file diff --git a/src/vs/workbench/parts/search/browser/openFileHandler.ts b/src/vs/workbench/parts/search/browser/openFileHandler.ts index 9e4e49263eb..4ac841c8e41 100644 --- a/src/vs/workbench/parts/search/browser/openFileHandler.ts +++ b/src/vs/workbench/parts/search/browser/openFileHandler.ts @@ -104,7 +104,7 @@ export class OpenFileHandler extends QuickOpenHandler { .then(result => result[0]); } - public getResultsWithStats(searchValue: string): TPromise<[QuickOpenModel, ISearchStats]> { + public getResultsWithStats(searchValue: string, cacheKey?: string, maxSortedResults?: number): TPromise<[QuickOpenModel, ISearchStats]> { searchValue = searchValue.trim(); let promise: TPromise<[QuickOpenEntry[], ISearchStats]>; @@ -112,18 +112,23 @@ export class OpenFileHandler extends QuickOpenHandler { if (!searchValue) { promise = TPromise.as(<[QuickOpenEntry[], ISearchStats]>[[], undefined]); } else { - promise = this.doFindResults(searchValue); + promise = this.doFindResults(searchValue, cacheKey, maxSortedResults); } return promise.then(result => [new QuickOpenModel(result[0]), result[1]]); } - private doFindResults(searchValue: string): TPromise<[QuickOpenEntry[], ISearchStats]> { + private doFindResults(searchValue: string, cacheKey?: string, maxSortedResults?: number): TPromise<[QuickOpenEntry[], ISearchStats]> { const query: IQueryOptions = { folderResources: this.contextService.getWorkspace() ? [this.contextService.getWorkspace().resource] : [], extraFileResources: getOutOfWorkspaceEditorResources(this.editorGroupService, this.contextService), - filePattern: searchValue + filePattern: searchValue, + cacheKey: cacheKey }; + if (maxSortedResults) { + query.maxResults = maxSortedResults; + query.sortByScore = true; + } return this.searchService.search(this.queryBuilder.file(query)).then((complete) => { let results: QuickOpenEntry[] = []; @@ -140,6 +145,10 @@ export class OpenFileHandler extends QuickOpenHandler { }); } + public clearCache(cacheKey: string): TPromise { + return this.searchService.clearCache(cacheKey); + } + public getGroupLabel(): string { return nls.localize('searchResults', "search results"); } diff --git a/src/vs/workbench/parts/search/common/searchQuery.ts b/src/vs/workbench/parts/search/common/searchQuery.ts index 39418133ff7..55e2b45ea85 100644 --- a/src/vs/workbench/parts/search/common/searchQuery.ts +++ b/src/vs/workbench/parts/search/common/searchQuery.ts @@ -59,6 +59,8 @@ export class QueryBuilder { excludePattern: options.excludePattern, includePattern: options.includePattern, maxResults: options.maxResults, + sortByScore: options.sortByScore, + cacheKey: options.cacheKey, fileEncoding: options.fileEncoding, contentPattern: contentPattern }; diff --git a/src/vs/workbench/parts/search/test/common/searchModel.test.ts b/src/vs/workbench/parts/search/test/common/searchModel.test.ts index 0ad7b9947e5..1f07543f1c4 100644 --- a/src/vs/workbench/parts/search/test/common/searchModel.test.ts +++ b/src/vs/workbench/parts/search/test/common/searchModel.test.ts @@ -14,7 +14,7 @@ import { SearchModel } from 'vs/workbench/parts/search/common/searchModel'; import URI from 'vs/base/common/uri'; import {IFileMatch, ILineMatch} from 'vs/platform/search/common/search'; import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry'; -import { ISearchService, ISearchComplete, ISearchProgressItem, ISearchStats } from 'vs/platform/search/common/search'; +import { ISearchService, ISearchComplete, ISearchProgressItem, IUncachedSearchStats } from 'vs/platform/search/common/search'; import { Range } from 'vs/editor/common/core/range'; import { createMockModelService } from 'vs/test/utils/servicesTestUtils'; import { IModelService } from 'vs/editor/common/services/modelService'; @@ -24,7 +24,9 @@ suite('SearchModel', () => { let instantiationService: TestInstantiationService; let restoreStubs; - const testSearchStats: ISearchStats = { + const testSearchStats: IUncachedSearchStats = { + fromCache: false, + resultCount: 4, fileWalkStartTime: 0, fileWalkResultTime: 1, directoriesWalked: 2, diff --git a/src/vs/workbench/services/search/node/fileSearch.ts b/src/vs/workbench/services/search/node/fileSearch.ts index 1061b7873d7..eef69c85423 100644 --- a/src/vs/workbench/services/search/node/fileSearch.ts +++ b/src/vs/workbench/services/search/node/fileSearch.ts @@ -13,11 +13,11 @@ import arrays = require('vs/base/common/arrays'); import strings = require('vs/base/common/strings'); import types = require('vs/base/common/types'); import glob = require('vs/base/common/glob'); -import {IProgress, ISearchStats} from 'vs/platform/search/common/search'; +import {IProgress, IUncachedSearchStats} from 'vs/platform/search/common/search'; import extfs = require('vs/base/node/extfs'); import flow = require('vs/base/node/flow'); -import {ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine} from './search'; +import {IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine} from './search'; export class FileWalker { private config: IRawSearch; @@ -59,7 +59,7 @@ export class FileWalker { this.isCanceled = true; } - public walk(rootFolders: string[], extraFiles: string[], onResult: (result: ISerializedFileMatch, size: number) => void, done: (error: Error, isLimitHit: boolean) => void): void { + public walk(rootFolders: string[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void { this.fileWalkStartTime = Date.now(); // Support that the file pattern is a full path to a file that exists @@ -70,7 +70,12 @@ export class FileWalker { // Report result from file pattern if matching if (exists) { - onResult({ path: this.filePattern }, size); + this.resultCount++; + onResult({ + absolutePath: this.filePattern, + pathLabel: this.filePattern, + size + }); // Optimization: a match on an absolute path is a good result and we do not // continue walking the entire root paths array for other matches because @@ -106,7 +111,12 @@ export class FileWalker { // Report result from file pattern if matching if (match) { - onResult({ path: match }, size); + this.resultCount++; + onResult({ + absolutePath: match, + pathLabel: this.filePattern, + size + }); } return this.doWalk(paths.normalize(absolutePath), '', files, onResult, perEntryCallback); @@ -118,12 +128,14 @@ export class FileWalker { }); } - public getStats(): ISearchStats { + public getStats(): IUncachedSearchStats { return { + fromCache: false, fileWalkStartTime: this.fileWalkStartTime, fileWalkResultTime: Date.now(), directoriesWalked: this.directoriesWalked, - filesWalked: this.filesWalked + filesWalked: this.filesWalked, + resultCount: this.resultCount }; } @@ -149,7 +161,7 @@ export class FileWalker { }); } - private doWalk(absolutePath: string, relativeParentPathWithSlashes: string, files: string[], onResult: (result: ISerializedFileMatch, size: number) => void, done: (error: Error, result: any) => void): void { + private doWalk(absolutePath: string, relativeParentPathWithSlashes: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, result: any) => void): void { // Execute tasks on each file in parallel to optimize throughput flow.parallel(files, (file: string, clb: (error: Error) => void): void => { @@ -242,7 +254,7 @@ export class FileWalker { }); } - private matchFile(onResult: (result: ISerializedFileMatch, size: number) => void, absolutePath: string, relativePathWithSlashes: string, size?: number): void { + private matchFile(onResult: (result: IRawFileMatch) => void, absolutePath: string, relativePathWithSlashes: string, size?: number): void { if (this.isFilePatternMatch(relativePathWithSlashes) && (!this.includePattern || glob.match(this.includePattern, relativePathWithSlashes))) { this.resultCount++; @@ -252,8 +264,10 @@ export class FileWalker { if (!this.isLimitHit) { onResult({ - path: absolutePath - }, size); + absolutePath, + pathLabel: relativePathWithSlashes, + size + }); } } } @@ -296,7 +310,7 @@ export class FileWalker { } } -export class Engine implements ISearchEngine { +export class Engine implements ISearchEngine { private rootFolders: string[]; private extraFiles: string[]; private walker: FileWalker; @@ -308,7 +322,7 @@ export class Engine implements ISearchEngine { this.walker = new FileWalker(config); } - public search(onResult: (result: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { + public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { this.walker.walk(this.rootFolders, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => { done(err, { limitHit: isLimitHit, diff --git a/src/vs/workbench/services/search/node/rawSearchService.ts b/src/vs/workbench/services/search/node/rawSearchService.ts index 1450766c17c..a2e77f3929c 100644 --- a/src/vs/workbench/services/search/node/rawSearchService.ts +++ b/src/vs/workbench/services/search/node/rawSearchService.ts @@ -10,20 +10,29 @@ import fs = require('fs'); import gracefulFs = require('graceful-fs'); gracefulFs.gracefulify(fs); -import {PPromise} from 'vs/base/common/winjs.base'; +import arrays = require('vs/base/common/arrays'); +import {compareByScore} from 'vs/base/common/comparers'; +import objects = require('vs/base/common/objects'); +import paths = require('vs/base/common/paths'); +import scorer = require('vs/base/common/scorer'); +import strings = require('vs/base/common/strings'); +import {PPromise, TPromise} from 'vs/base/common/winjs.base'; import {MAX_FILE_SIZE} from 'vs/platform/files/common/files'; import {FileWalker, Engine as FileSearchEngine} from 'vs/workbench/services/search/node/fileSearch'; import {Engine as TextSearchEngine} from 'vs/workbench/services/search/node/textSearch'; -import {IRawSearchService, IRawSearch, ISerializedSearchProgressItem, ISerializedSearchComplete, ISearchEngine} from './search'; +import {IRawSearchService, IRawSearch, IRawFileMatch, ISerializedFileMatch, ISerializedSearchProgressItem, ISerializedSearchComplete, ISearchEngine} from './search'; +import {ICachedSearchStats, IProgress} from 'vs/platform/search/common/search'; + +export type IRawProgressItem = T | T[] | IProgress; export class SearchService implements IRawSearchService { - private static BATCH_SIZE = 500; + private static BATCH_SIZE = 512; - public fileSearch(config: IRawSearch): PPromise { - let engine = new FileSearchEngine(config); + private caches: { [cacheKey: string]: Cache; } = Object.create(null); - return this.doSearch(engine, SearchService.BATCH_SIZE); + public fileSearch(config: IRawSearch): PPromise { + return this.doFileSearch(FileSearchEngine, config, SearchService.BATCH_SIZE); } public textSearch(config: IRawSearch): PPromise { @@ -39,8 +48,194 @@ export class SearchService implements IRawSearchService { return this.doSearch(engine, SearchService.BATCH_SIZE); } - public doSearch(engine: ISearchEngine, batchSize?: number): PPromise { + public doFileSearch(EngineClass: { new (config: IRawSearch): ISearchEngine; }, config: IRawSearch, batchSize?: number): PPromise { + + if (config.sortByScore) { + const cached = this.trySearchFromCache(config, batchSize); + if (cached) { + return cached; + } + + const walkerConfig = config.maxResults ? objects.assign({}, config, { maxResults: null }) : config; + const engine = new EngineClass(walkerConfig); + return this.doSortedSearch(engine, config, batchSize); + } + + let searchPromise; return new PPromise((c, e, p) => { + const engine = new EngineClass(config); + searchPromise = this.doSearch(engine, batchSize) + .then(c, e, progress => { + if (Array.isArray(progress)) { + p(progress.map(m => ({ path: m.absolutePath }))); + } else if ((progress).absolutePath) { + p({ path: (progress).absolutePath }); + } else { + p(progress); + } + }); + }, () => searchPromise.cancel()); + } + + private doSortedSearch(engine: ISearchEngine, config: IRawSearch, batchSize?: number): PPromise> { + let searchPromise; + return new PPromise((c, e, p) => { + let results: IRawFileMatch[] = []; + let unsortedResultTime: number; + let sortedResultTime: number; + searchPromise = this.doSearch(engine, -1).then(result => { + const maxResults = config.maxResults; + result.limitHit = !!maxResults && results.length > maxResults; + result.stats.unsortedResultTime = unsortedResultTime || Date.now(); + result.stats.sortedResultTime = sortedResultTime || Date.now(); + c(result); + }, null, progress => { + try { + if (Array.isArray(progress)) { + results = progress; + let scorerCache; + if (config.cacheKey) { + const cache = this.getOrCreateCache(config.cacheKey); + cache.resultsToSearchCache[config.filePattern] = results; + scorerCache = cache.scorerCache; + } else { + scorerCache = Object.create(null); + } + unsortedResultTime = Date.now(); + const sortedResults = this.sortResults(config, results, scorerCache); + sortedResultTime = Date.now(); + this.sendProgress(sortedResults, p, batchSize); + } else { + p(progress); + } + } catch (err) { + e(err); + } + }).then(null, e); + }, () => searchPromise.cancel()); + } + + private getOrCreateCache(cacheKey: string): Cache { + const existing = this.caches[cacheKey]; + if (existing) { + return existing; + } + return this.caches[cacheKey] = new Cache(); + } + + private trySearchFromCache(config: IRawSearch, batchSize?: number): PPromise { + const cache = config.cacheKey && this.caches[config.cacheKey]; + if (!cache) { + return; + } + + const cacheLookupStartTime = Date.now(); + const cached = this.getResultsFromCache(cache, config.filePattern); + if (cached) { + const cacheLookupResultTime = Date.now(); + const [results, cacheEntryCount] = cached; + let clippedResults; + if (config.sortByScore) { + clippedResults = this.sortResults(config, results, cache); + } else if (config.maxResults) { + clippedResults = results.slice(0, config.maxResults); + } else { + clippedResults = results; + } + const sortedResultTime = Date.now(); + let canceled = false; + return new PPromise((c, e, p) => { + process.nextTick(() => { // allow caller to register progress callback first + if (canceled) { + return; + } + this.sendProgress(clippedResults, p, batchSize); + const maxResults = config.maxResults; + const stats: ICachedSearchStats = { + fromCache: true, + cacheLookupStartTime: cacheLookupStartTime, + cacheLookupResultTime: cacheLookupResultTime, + cacheEntryCount: cacheEntryCount, + resultCount: results.length + }; + if (config.sortByScore) { + stats.unsortedResultTime = cacheLookupResultTime; + stats.sortedResultTime = sortedResultTime; + } + c({ + limitHit: !!maxResults && results.length > maxResults, + stats: stats + }); + }); + }, () => { + canceled = true; + }); + } + } + + private sortResults(config: IRawSearch, results: IRawFileMatch[], cache: Cache): ISerializedFileMatch[] { + const filePattern = config.filePattern; + const normalizedSearchValue = strings.stripWildcards(filePattern).toLowerCase(); + const compare = (elementA: IRawFileMatch, elementB: IRawFileMatch) => compareByScore(elementA, elementB, FileMatchAccessor, filePattern, normalizedSearchValue, cache.scorerCache); + const filteredWrappers = arrays.top(results, compare, config.maxResults); + return filteredWrappers.map(result => ({ path: result.absolutePath })); + } + + private sendProgress(results: ISerializedFileMatch[], progressCb: (batch: ISerializedFileMatch[]) => void, batchSize?: number) { + if (batchSize && batchSize > 0) { + for (let i = 0; i < results.length; i += batchSize) { + progressCb(results.slice(i, i + batchSize)); + } + } else { + progressCb(results); + } + } + + private getResultsFromCache(cache: Cache, searchValue: string): [IRawFileMatch[], number] { + if (paths.isAbsolute(searchValue)) { + return null; // bypass cache if user looks up an absolute path where matching goes directly on disk + } + + // Find cache entries by prefix of search value + const hasPathSep = searchValue.indexOf(paths.nativeSep) >= 0; + let cachedEntries: IRawFileMatch[]; + for (let previousSearch in cache.resultsToSearchCache) { + + // If we narrow down, we might be able to reuse the cached results + if (strings.startsWith(searchValue, previousSearch)) { + if (hasPathSep && previousSearch.indexOf(paths.nativeSep) < 0) { + continue; // since a path character widens the search for potential more matches, require it in previous search too + } + + cachedEntries = cache.resultsToSearchCache[previousSearch]; + break; + } + } + + if (!cachedEntries) { + return null; + } + + // Pattern match on results and adjust highlights + let results: IRawFileMatch[] = []; + const normalizedSearchValue = searchValue.replace(/\\/g, '/'); // Normalize file patterns to forward slashes + const normalizedSearchValueLowercase = strings.stripWildcards(normalizedSearchValue).toLowerCase(); + for (let i = 0; i < cachedEntries.length; i++) { + let entry = cachedEntries[i]; + + // Check if this entry is a match for the search value + if (!scorer.matches(entry.pathLabel, normalizedSearchValueLowercase)) { + continue; + } + + results.push(entry); + } + + return [results, cachedEntries.length]; + } + + private doSearch(engine: ISearchEngine, batchSize?: number): PPromise> { + return new PPromise>((c, e, p) => { let batch = []; engine.search((match) => { if (match) { @@ -68,4 +263,34 @@ export class SearchService implements IRawSearchService { }); }, () => engine.cancel()); } -} \ No newline at end of file + + public clearCache(cacheKey: string): TPromise { + delete this.caches[cacheKey]; + return TPromise.as(undefined); + } +} + +class Cache { + + public resultsToSearchCache: { [searchValue: string]: IRawFileMatch[]; } = Object.create(null); + + public scorerCache: { [key: string]: number } = Object.create(null); +} + +interface IFileMatch extends IRawFileMatch { + label?: string; +} + +class FileMatchAccessor { + + public static getLabel(match: IFileMatch): string { + if (!match.label) { + match.label = paths.basename(match.absolutePath); + } + return match.label; + } + + public static getResourcePath(match: IFileMatch): string { + return match.absolutePath; + } +} diff --git a/src/vs/workbench/services/search/node/search.ts b/src/vs/workbench/services/search/node/search.ts index 1f8cb397188..765effe2768 100644 --- a/src/vs/workbench/services/search/node/search.ts +++ b/src/vs/workbench/services/search/node/search.ts @@ -5,7 +5,7 @@ 'use strict'; -import { PPromise } from 'vs/base/common/winjs.base'; +import { PPromise, TPromise } from 'vs/base/common/winjs.base'; import glob = require('vs/base/common/glob'); import { IProgress, ILineMatch, IPatternInfo, ISearchStats } from 'vs/platform/search/common/search'; @@ -17,6 +17,8 @@ export interface IRawSearch { includePattern?: glob.IExpression; contentPattern?: IPatternInfo; maxResults?: number; + sortByScore?: boolean; + cacheKey?: string; maxFilesize?: number; fileEncoding?: string; } @@ -24,10 +26,17 @@ export interface IRawSearch { export interface IRawSearchService { fileSearch(search: IRawSearch): PPromise; textSearch(search: IRawSearch): PPromise; + clearCache(cacheKey: string): TPromise; } -export interface ISearchEngine { - search: (onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void) => void; +export interface IRawFileMatch { + absolutePath: string; + pathLabel: string; + size: number; +} + +export interface ISearchEngine { + search: (onResult: (match: T) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void) => void; cancel: () => void; } diff --git a/src/vs/workbench/services/search/node/searchIpc.ts b/src/vs/workbench/services/search/node/searchIpc.ts index 027c1461cb3..fd5eacdefa6 100644 --- a/src/vs/workbench/services/search/node/searchIpc.ts +++ b/src/vs/workbench/services/search/node/searchIpc.ts @@ -12,6 +12,7 @@ import { IRawSearchService, IRawSearch, ISerializedSearchComplete, ISerializedSe export interface ISearchChannel extends IChannel { call(command: 'fileSearch', search: IRawSearch): PPromise; call(command: 'textSearch', search: IRawSearch): PPromise; + call(command: 'clearCache', cacheKey: string): TPromise; call(command: string, arg: any): TPromise; } @@ -23,6 +24,7 @@ export class SearchChannel implements ISearchChannel { switch (command) { case 'fileSearch': return this.service.fileSearch(arg); case 'textSearch': return this.service.textSearch(arg); + case 'clearCache': return this.service.clearCache(arg); } } } @@ -38,4 +40,8 @@ export class SearchChannelClient implements IRawSearchService { textSearch(search: IRawSearch): PPromise { return this.channel.call('textSearch', search); } + + public clearCache(cacheKey: string): TPromise { + return this.channel.call('clearCache', cacheKey); + } } \ No newline at end of file diff --git a/src/vs/workbench/services/search/node/searchService.ts b/src/vs/workbench/services/search/node/searchService.ts index 737622808d0..ba5197fa8b6 100644 --- a/src/vs/workbench/services/search/node/searchService.ts +++ b/src/vs/workbench/services/search/node/searchService.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {PPromise} from 'vs/base/common/winjs.base'; +import {PPromise, TPromise} from 'vs/base/common/winjs.base'; import uri from 'vs/base/common/uri'; import glob = require('vs/base/common/glob'); import objects = require('vs/base/common/objects'); @@ -189,6 +189,10 @@ export class SearchService implements ISearchService { return true; } + + public clearCache(cacheKey: string): TPromise { + return this.diskSearch.clearCache(cacheKey); + } } export class DiskSearch { @@ -223,7 +227,9 @@ export class DiskSearch { filePattern: query.filePattern, excludePattern: query.excludePattern, includePattern: query.includePattern, - maxResults: query.maxResults + maxResults: query.maxResults, + sortByScore: query.sortByScore, + cacheKey: query.cacheKey }; if (query.type === QueryType.Text) { @@ -282,4 +288,8 @@ export class DiskSearch { } return fileMatch; } + + public clearCache(cacheKey: string): TPromise { + return this.raw.clearCache(cacheKey); + } } \ No newline at end of file diff --git a/src/vs/workbench/services/search/node/textSearch.ts b/src/vs/workbench/services/search/node/textSearch.ts index a4735e13d32..4b44fefb8bb 100644 --- a/src/vs/workbench/services/search/node/textSearch.ts +++ b/src/vs/workbench/services/search/node/textSearch.ts @@ -21,7 +21,7 @@ interface ReadLinesOptions { encoding: string; } -export class Engine implements ISearchEngine { +export class Engine implements ISearchEngine { private static PROGRESS_FLUSH_CHUNK_SIZE = 50; // optimization: number of files to process before emitting progress event @@ -88,8 +88,8 @@ export class Engine implements ISearchEngine { }; // Walk over the file system - this.walker.walk(this.rootFolders, this.extraFiles, (result, size) => { - size = size ||  1; + this.walker.walk(this.rootFolders, this.extraFiles, result => { + const size = result.size ||  1; this.total += size; // If the result is empty or we have reached the limit or we are canceled, ignore it @@ -126,7 +126,7 @@ export class Engine implements ISearchEngine { } if (fileMatch === null) { - fileMatch = new FileMatch(result.path); + fileMatch = new FileMatch(result.absolutePath); } if (lineMatch === null) { @@ -141,7 +141,7 @@ export class Engine implements ISearchEngine { }; // Read lines buffered to support large files - this.readlinesAsync(result.path, perLineCallback, { bufferLength: 8096, encoding: this.fileEncoding }, doneCallback); + this.readlinesAsync(result.absolutePath, perLineCallback, { bufferLength: 8096, encoding: this.fileEncoding }, doneCallback); }, (error, isLimitHit) => { this.walkerIsDone = true; this.walkerError = error; diff --git a/src/vs/workbench/services/search/test/node/search.test.ts b/src/vs/workbench/services/search/test/node/search.test.ts index f3fc9613a4a..1685facef39 100644 --- a/src/vs/workbench/services/search/test/node/search.test.ts +++ b/src/vs/workbench/services/search/test/node/search.test.ts @@ -12,6 +12,7 @@ import {join, normalize} from 'vs/base/common/paths'; import {LineMatch} from 'vs/platform/search/common/search'; import {FileWalker, Engine as FileSearchEngine} from 'vs/workbench/services/search/node/fileSearch'; +import {IRawFileMatch} from 'vs/workbench/services/search/node/search'; import {Engine as TextSearchEngine} from 'vs/workbench/services/search/node/textSearch'; function count(lineMatches: LineMatch[]): number { @@ -149,7 +150,7 @@ suite('Search', () => { }); let count = 0; - let res; + let res: IRawFileMatch; engine.search((result) => { if (result) { count++; @@ -158,7 +159,7 @@ suite('Search', () => { }, () => { }, (error) => { assert.ok(!error); assert.equal(count, 1); - assert.ok(path.basename(res.path) === 'site.less'); + assert.strictEqual(path.basename(res.absolutePath), 'site.less'); done(); }); }); @@ -246,7 +247,7 @@ suite('Search', () => { }); let count = 0; - let res; + let res: IRawFileMatch; engine.search((result) => { if (result) { count++; @@ -255,7 +256,7 @@ suite('Search', () => { }, () => { }, (error) => { assert.ok(!error); assert.equal(count, 1); - assert.equal(path.basename(res.path), '汉语.txt'); + assert.equal(path.basename(res.absolutePath), '汉语.txt'); done(); }); }); @@ -286,7 +287,7 @@ suite('Search', () => { }); let count = 0; - let res; + let res: IRawFileMatch; engine.search((result) => { if (result) { count++; @@ -295,7 +296,7 @@ suite('Search', () => { }, () => { }, (error) => { assert.ok(!error); assert.equal(count, 1); - assert.equal(path.basename(res.path), 'site.css'); + assert.equal(path.basename(res.absolutePath), 'site.css'); done(); }); }); @@ -308,7 +309,7 @@ suite('Search', () => { }); let count = 0; - let res; + let res: IRawFileMatch; engine.search((result) => { if (result) { count++; @@ -317,7 +318,7 @@ suite('Search', () => { }, () => { }, (error) => { assert.ok(!error); assert.equal(count, 1); - assert.equal(path.basename(res.path), 'company.js'); + assert.equal(path.basename(res.absolutePath), 'company.js'); done(); }); }); @@ -334,7 +335,7 @@ suite('Search', () => { }); let count = 0; - let res; + let res: IRawFileMatch; engine.search((result) => { if (result) { count++; @@ -343,7 +344,7 @@ suite('Search', () => { }, () => { }, (error) => { assert.ok(!error); assert.equal(count, 1); - assert.equal(path.basename(res.path), 'company.js'); + assert.equal(path.basename(res.absolutePath), 'company.js'); done(); }); }); @@ -361,7 +362,7 @@ suite('Search', () => { }); let count = 0; - let res; + let res: IRawFileMatch; engine.search((result) => { if (result) { count++; @@ -370,7 +371,7 @@ suite('Search', () => { }, () => { }, (error) => { assert.ok(!error); assert.equal(count, 1); - assert.equal(path.basename(res.path), 'site.css'); + assert.equal(path.basename(res.absolutePath), 'site.css'); done(); }); }); diff --git a/src/vs/workbench/services/search/test/node/searchService.test.ts b/src/vs/workbench/services/search/test/node/searchService.test.ts index fd341943d70..5cafb9e7304 100644 --- a/src/vs/workbench/services/search/test/node/searchService.test.ts +++ b/src/vs/workbench/services/search/test/node/searchService.test.ts @@ -7,31 +7,47 @@ import assert = require('assert'); -import {IProgress} from 'vs/platform/search/common/search'; -import {ISearchEngine, ISerializedFileMatch, ISerializedSearchComplete} from 'vs/workbench/services/search/node/search'; +import {IProgress, IUncachedSearchStats} from 'vs/platform/search/common/search'; +import {ISearchEngine, IRawSearch, IRawFileMatch, ISerializedFileMatch, ISerializedSearchComplete} from 'vs/workbench/services/search/node/search'; import {SearchService as RawSearchService} from 'vs/workbench/services/search/node/rawSearchService'; import {DiskSearch} from 'vs/workbench/services/search/node/searchService'; -class TestSearchEngine implements ISearchEngine { +const stats: IUncachedSearchStats = { + fromCache: false, + resultCount: 4, + fileWalkStartTime: 0, + fileWalkResultTime: 1, + directoriesWalked: 2, + filesWalked: 3 +}; - constructor(private result: () => ISerializedFileMatch) { +class TestSearchEngine implements ISearchEngine { + + public static last: TestSearchEngine; + + private isCanceled = false; + + constructor(private result: () => IRawFileMatch, public config?: IRawSearch) { + TestSearchEngine.last = this; } - public search(onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { + public search(onResult: (match: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { const self = this; (function next() { process.nextTick(() => { + if (self.isCanceled) { + done(null, { + limitHit: false, + stats: stats + }); + return; + } const result = self.result(); if (!result) { done(null, { limitHit: false, - stats: { - fileWalkStartTime: 0, - fileWalkResultTime: 1, - directoriesWalked: 2, - filesWalked: 3 - } + stats: stats }); } else { onResult(result); @@ -42,24 +58,39 @@ class TestSearchEngine implements ISearchEngine { } public cancel(): void { + this.isCanceled = true; } } suite('SearchService', () => { + const rawSearch: IRawSearch = { + rootFolders: ['/some/where'], + filePattern: 'a' + }; + + const rawMatch: IRawFileMatch = { + absolutePath: '/some/where', + pathLabel: 'where', + size: 123 + }; + + const match: ISerializedFileMatch = { + path: '/some/where' + }; + test('Individual results', function () { - const path = '/some/where'; let i = 5; - const engine = new TestSearchEngine(() => i-- && { path }); + const Engine = TestSearchEngine.bind(null, () => i-- && rawMatch); const service = new RawSearchService(); let results = 0; - return service.doSearch(engine) + return service.doFileSearch(Engine, rawSearch) .then(() => { assert.strictEqual(results, 5); }, null, value => { if (!Array.isArray(value)) { - assert.strictEqual((value).path, path); + assert.deepStrictEqual(value, match); results++; } else { assert.fail(value); @@ -68,19 +99,18 @@ suite('SearchService', () => { }); test('Batch results', function () { - const path = '/some/where'; let i = 25; - const engine = new TestSearchEngine(() => i-- && { path }); + const Engine = TestSearchEngine.bind(null, () => i-- && rawMatch); const service = new RawSearchService(); const results = []; - return service.doSearch(engine, 10) + return service.doFileSearch(Engine, rawSearch, 10) .then(() => { assert.deepStrictEqual(results, [10, 10, 5]); }, null, value => { if (Array.isArray(value)) { - value.forEach(match => { - assert.strictEqual(match.path, path); + value.forEach(m => { + assert.deepStrictEqual(m, match); }); results.push(value.length); } else { @@ -92,12 +122,12 @@ suite('SearchService', () => { test('Collect batched results', function () { const path = '/some/where'; let i = 25; - const engine = new TestSearchEngine(() => i-- && { path }); + const Engine = TestSearchEngine.bind(null, () => i-- && rawMatch); const service = new RawSearchService(); const diskSearch = new DiskSearch(false); const progressResults = []; - return DiskSearch.collectResults(service.doSearch(engine, 10)) + return DiskSearch.collectResults(service.doFileSearch(Engine, rawSearch, 10)) .then(result => { assert.strictEqual(result.results.length, 25, 'Result'); assert.strictEqual(progressResults.length, 25, 'Progress'); @@ -106,4 +136,129 @@ suite('SearchService', () => { progressResults.push(match); }); }); + + test('Sorted results', function () { + const paths = ['bab', 'bbc', 'abb']; + const matches = paths.map(path => ({ + absolutePath: `/some/where/${path}`, + pathLabel: path, + size: 3 + })); + let i = 0; + const Engine = TestSearchEngine.bind(null, () => matches.shift()); + const service = new RawSearchService(); + + const results = []; + return service.doFileSearch(Engine, { + rootFolders: ['/some/where'], + filePattern: 'bb', + sortByScore: true, + maxResults: 2 + }, 1).then(() => { + assert.notStrictEqual(typeof TestSearchEngine.last.config.maxResults, 'number'); + assert.deepStrictEqual(results, ['/some/where/bbc', '/some/where/bab']); + }, null, value => { + if (Array.isArray(value)) { + results.push(...value.map(v => v.path)); + } else { + assert.fail(value); + } + }); + }); + + test('Sorted result batches', function () { + let i = 25; + const Engine = TestSearchEngine.bind(null, () => i-- && rawMatch); + const service = new RawSearchService(); + + const results = []; + return service.doFileSearch(Engine, { + rootFolders: ['/some/where'], + filePattern: 'a', + sortByScore: true, + maxResults: 23 + }, 10) + .then(() => { + assert.deepStrictEqual(results, [10, 10, 3]); + }, null, value => { + if (Array.isArray(value)) { + value.forEach(m => { + assert.deepStrictEqual(m, match); + }); + results.push(value.length); + } else { + assert.fail(value); + } + }); + }); + + test('Cached results', function () { + const paths = ['bcb', 'bbc', 'aab']; + const matches = paths.map(path => ({ + absolutePath: `/some/where/${path}`, + pathLabel: path, + size: 3 + })); + let i = 0; + const Engine = TestSearchEngine.bind(null, () => matches.shift()); + const service = new RawSearchService(); + + const results = []; + return service.doFileSearch(Engine, { + rootFolders: ['/some/where'], + filePattern: 'b', + sortByScore: true, + cacheKey: 'x' + }, -1).then(complete => { + assert.strictEqual(complete.stats.fromCache, false); + assert.deepStrictEqual(results, ['/some/where/bcb', '/some/where/bbc', '/some/where/aab']); + }, null, value => { + if (Array.isArray(value)) { + results.push(...value.map(v => v.path)); + } else { + assert.fail(value); + } + }).then(() => { + const results = []; + return service.doFileSearch(Engine, { + rootFolders: ['/some/where'], + filePattern: 'bc', + sortByScore: true, + cacheKey: 'x' + }, -1).then(complete => { + assert.ok(complete.stats.fromCache); + assert.deepStrictEqual(results, ['/some/where/bcb', '/some/where/bbc']); + }, null, value => { + if (Array.isArray(value)) { + results.push(...value.map(v => v.path)); + } else { + assert.fail(value); + } + }); + }).then(() => { + return service.clearCache('x'); + }).then(() => { + matches.push({ + absolutePath: '/some/where/bc', + pathLabel: 'bc', + size: 3 + }); + const results = []; + return service.doFileSearch(Engine, { + rootFolders: ['/some/where'], + filePattern: 'bc', + sortByScore: true, + cacheKey: 'x' + }, -1).then(complete => { + assert.strictEqual(complete.stats.fromCache, false); + assert.deepStrictEqual(results, ['/some/where/bc']); + }, null, value => { + if (Array.isArray(value)) { + results.push(...value.map(v => v.path)); + } else { + assert.fail(value); + } + }); + }); + }); }); \ No newline at end of file diff --git a/src/vs/workbench/test/browser/parts/quickOpen/quickopen.perf.test.ts b/src/vs/workbench/test/browser/parts/quickOpen/quickopen.perf.test.ts index c38b929dac1..1ba24604d2a 100644 --- a/src/vs/workbench/test/browser/parts/quickOpen/quickopen.perf.test.ts +++ b/src/vs/workbench/test/browser/parts/quickOpen/quickopen.perf.test.ts @@ -36,7 +36,9 @@ suite('QuickOpen performance', () => { test('Measure', () => { const n = 3; - const testWorkspaceArg = minimist(process.argv)['testWorkspace']; + const argv = minimist(process.argv); + const testWorkspaceArg = argv['testWorkspace']; + const verboseResults = argv['verboseResults']; const testWorkspacePath = testWorkspaceArg ? path.join(process.cwd(), testWorkspaceArg) : __dirname; const telemetryService = new TestTelemetryService(); @@ -69,12 +71,15 @@ suite('QuickOpen performance', () => { .then((handler: QuickOpenHandler) => { return handler.getResults('a').then(result => { const uncachedEvent = popEvent(); - assert.ok(!uncachedEvent.data.fromCache); + assert.strictEqual(uncachedEvent.data.symbols.fromCache, false, 'symbols.fromCache'); + assert.strictEqual(uncachedEvent.data.files.fromCache, false, 'files.fromCache'); return uncachedEvent; }).then(uncachedEvent => { return handler.getResults('ab').then(result => { const cachedEvent = popEvent(); - assert.ok(cachedEvent.data.fromCache); + assert.ok(cachedEvent.data.symbols.fromCache, 'symbolsFromCache'); + assert.ok(cachedEvent.data.files.fromCache, 'filesFromCache'); + handler.onClose(false); return [uncachedEvent, cachedEvent]; }); }); @@ -90,6 +95,19 @@ suite('QuickOpen performance', () => { return event; } + function printResult(data) { + if (verboseResults) { + console.log(JSON.stringify(data, null, ' ') + ','); + } else { + console.log(JSON.stringify({ + filesfromCache: data.files.fromCache, + searchLength: data.searchLength, + sortedResultDuration: data.sortedResultDuration, + filesResultCount: data.files.resultCount, + }) + ','); + } + } + return measure() // Warm-up first .then(() => { if (testWorkspaceArg) { // Don't measure by default @@ -101,14 +119,14 @@ suite('QuickOpen performance', () => { } return measure() .then(([uncachedEvent, cachedEvent]) => { - console.log(JSON.stringify(uncachedEvent.data) + ','); + printResult(uncachedEvent.data); cachedEvents.push(cachedEvent); return iterate(); }); })().then(() => { console.log(); cachedEvents.forEach(cachedEvent => { - console.log(JSON.stringify(cachedEvent.data) + ','); + printResult(cachedEvent.data); }); }); } -- GitLab