From a19857db3dd522cb2d2f93c60206bbd71eab0144 Mon Sep 17 00:00:00 2001 From: Benjamin Pasero Date: Tue, 31 Mar 2020 09:54:25 +0200 Subject: [PATCH] scorer - introduce normalized path --- src/vs/base/common/fuzzyScorer.ts | 80 ++++++++++++------- src/vs/base/test/common/fuzzyScorer.test.ts | 47 +++++++---- .../browser/parts/editor/editorQuickAccess.ts | 4 +- .../search/browser/anythingQuickAccess.ts | 14 ++-- .../services/search/node/fileSearch.ts | 2 +- .../services/search/node/rawSearchService.ts | 2 +- 6 files changed, 96 insertions(+), 53 deletions(-) diff --git a/src/vs/base/common/fuzzyScorer.ts b/src/vs/base/common/fuzzyScorer.ts index dd74aed4721..7ffa709f589 100644 --- a/src/vs/base/common/fuzzyScorer.ts +++ b/src/vs/base/common/fuzzyScorer.ts @@ -25,15 +25,15 @@ export function score(target: string, query: IPreparedQuery, fuzzy: boolean): Sc return scoreMultiple(target, query.values, fuzzy); } - return scoreSingle(target, query.value, query.valueLowercase, fuzzy); + return scoreSingle(target, query.normalized, query.normalizedLowercase, fuzzy); } function scoreMultiple(target: string, query: IPreparedQueryPiece[], fuzzy: boolean): Score { let totalScore = NO_MATCH; const totalPositions: number[] = []; - for (const { value, valueLowercase } of query) { - const [scoreValue, positions] = scoreSingle(target, value, valueLowercase, fuzzy); + for (const { normalized, normalizedLowercase } of query) { + const [scoreValue, positions] = scoreSingle(target, normalized, normalizedLowercase, fuzzy); if (scoreValue === NO_MATCH) { // if a single query value does not match, return with // no score entirely, we require all queries to match @@ -338,11 +338,26 @@ const LABEL_CAMELCASE_SCORE = 1 << 16; const LABEL_SCORE_THRESHOLD = 1 << 15; export interface IPreparedQueryPiece { + + /** + * The original query as provided as input. + */ original: string; originalLowercase: string; - value: string; - valueLowercase: string; + /** + * Original normalized to platform separators: + * - Windows: \ + * - Posix: / + */ + pathNormalized: string; + + /** + * In addition to the normalized path, will have + * whitespace and wildcards removed. + */ + normalized: string; + normalizedLowercase: string; } export interface IPreparedQuery extends IPreparedQueryPiece { @@ -364,17 +379,21 @@ export function prepareQuery(original: string): IPreparedQuery { } const originalLowercase = original.toLowerCase(); - const value = prepareQueryValue(original); - const valueLowercase = value.toLowerCase(); - const containsPathSeparator = value.indexOf(sep) >= 0; + const { pathNormalized, normalized, normalizedLowercase } = normalizeQuery(original); + const containsPathSeparator = pathNormalized.indexOf(sep) >= 0; let values: IPreparedQueryPiece[] | undefined = undefined; const originalSplit = original.split(MULTIPL_QUERY_VALUES_SEPARATOR); if (originalSplit.length > 1) { for (const originalPiece of originalSplit) { - const valuePiece = prepareQueryValue(originalPiece); - if (valuePiece) { + const { + pathNormalized: pathNormalizedPiece, + normalized: normalizedPiece, + normalizedLowercase: normalizedLowercasePiece + } = normalizeQuery(originalPiece); + + if (normalizedPiece) { if (!values) { values = []; } @@ -382,29 +401,36 @@ export function prepareQuery(original: string): IPreparedQuery { values.push({ original: originalPiece, originalLowercase: originalPiece.toLowerCase(), - value: valuePiece, - valueLowercase: valuePiece.toLowerCase() + pathNormalized: pathNormalizedPiece, + normalized: normalizedPiece, + normalizedLowercase: normalizedLowercasePiece }); } } } - return { original, originalLowercase, value, valueLowercase, values, containsPathSeparator }; + return { original, originalLowercase, pathNormalized, normalized, normalizedLowercase, values, containsPathSeparator }; } -function prepareQueryValue(original: string): string { - let value = stripWildcards(original).replace(/\s/g, ''); // get rid of all wildcards and whitespace +function normalizeQuery(original: string): { pathNormalized: string, normalized: string, normalizedLowercase: string } { + let pathNormalized: string; if (isWindows) { - value = value.replace(/\//g, sep); // Help Windows users to search for paths when using slash + pathNormalized = original.replace(/\//g, sep); // Help Windows users to search for paths when using slash } else { - value = value.replace(/\\/g, sep); // Help macOS/Linux users to search for paths when using backslash + pathNormalized = original.replace(/\\/g, sep); // Help macOS/Linux users to search for paths when using backslash } - return value; + const normalized = stripWildcards(pathNormalized).replace(/\s/g, ''); + + return { + pathNormalized, + normalized, + normalizedLowercase: normalized.toLowerCase() + }; } export function scoreItem(item: T, query: IPreparedQuery, fuzzy: boolean, accessor: IItemAccessor, cache: ScorerCache): IItemScore { - if (!item || !query.value) { + if (!item || !query.normalized) { return NO_ITEM_SCORE; // we need an item and query to score on at least } @@ -417,9 +443,9 @@ export function scoreItem(item: T, query: IPreparedQuery, fuzzy: boolean, acc let cacheHash: string; if (description) { - cacheHash = `${label}${description}${query.value}${fuzzy}`; + cacheHash = `${label}${description}${query.normalized}${fuzzy}`; } else { - cacheHash = `${label}${query.value}${fuzzy}`; + cacheHash = `${label}${query.normalized}${fuzzy}`; } const cached = cache[cacheHash]; @@ -455,7 +481,7 @@ function createMatches(offsets: undefined | number[]): IMatch[] { function doScoreItem(label: string, description: string | undefined, path: string | undefined, query: IPreparedQuery, fuzzy: boolean): IItemScore { // 1.) treat identity matches on full path highest - if (path && (isLinux ? query.original === path : equalsIgnoreCase(query.original, path))) { + if (path && (isLinux ? query.pathNormalized === path : equalsIgnoreCase(query.pathNormalized, path))) { return { score: PATH_IDENTITY_SCORE, labelMatch: [{ start: 0, end: label.length }], descriptionMatch: description ? [{ start: 0, end: description.length }] : undefined }; } @@ -464,13 +490,13 @@ function doScoreItem(label: string, description: string | undefined, path: strin if (preferLabelMatches) { // 2.) treat prefix matches on the label second highest - const prefixLabelMatch = matchesPrefix(query.value, label); + const prefixLabelMatch = matchesPrefix(query.normalized, label); if (prefixLabelMatch) { return { score: LABEL_PREFIX_SCORE, labelMatch: prefixLabelMatch }; } // 3.) treat camelcase matches on the label third highest - const camelcaseLabelMatch = matchesCamelCase(query.value, label); + const camelcaseLabelMatch = matchesCamelCase(query.normalized, label); if (camelcaseLabelMatch) { return { score: LABEL_CAMELCASE_SCORE, labelMatch: camelcaseLabelMatch }; } @@ -702,17 +728,17 @@ function fallbackCompare(itemA: T, itemB: T, query: IPreparedQuery, accessor: // compare by label if (labelA !== labelB) { - return compareAnything(labelA, labelB, query.value); + return compareAnything(labelA, labelB, query.normalized); } // compare by description if (descriptionA && descriptionB && descriptionA !== descriptionB) { - return compareAnything(descriptionA, descriptionB, query.value); + return compareAnything(descriptionA, descriptionB, query.normalized); } // compare by path if (pathA && pathB && pathA !== pathB) { - return compareAnything(pathA, pathB, query.value); + return compareAnything(pathA, pathB, query.normalized); } // equal diff --git a/src/vs/base/test/common/fuzzyScorer.test.ts b/src/vs/base/test/common/fuzzyScorer.test.ts index dacbaa6f11f..39b4926546c 100644 --- a/src/vs/base/test/common/fuzzyScorer.test.ts +++ b/src/vs/base/test/common/fuzzyScorer.test.ts @@ -857,41 +857,58 @@ suite('Fuzzy Scorer', () => { }); test('prepareQuery', () => { - assert.equal(scorer.prepareQuery(' f*a ').value, 'fa'); + assert.equal(scorer.prepareQuery(' f*a ').normalized, 'fa'); assert.equal(scorer.prepareQuery('model Tester.ts').original, 'model Tester.ts'); assert.equal(scorer.prepareQuery('model Tester.ts').originalLowercase, 'model Tester.ts'.toLowerCase()); - assert.equal(scorer.prepareQuery('model Tester.ts').value, 'modelTester.ts'); - assert.equal(scorer.prepareQuery('Model Tester.ts').valueLowercase, 'modeltester.ts'); + assert.equal(scorer.prepareQuery('model Tester.ts').normalized, 'modelTester.ts'); + assert.equal(scorer.prepareQuery('Model Tester.ts').normalizedLowercase, 'modeltester.ts'); assert.equal(scorer.prepareQuery('ModelTester.ts').containsPathSeparator, false); assert.equal(scorer.prepareQuery('Model' + sep + 'Tester.ts').containsPathSeparator, true); // with spaces let query = scorer.prepareQuery('He*llo World'); assert.equal(query.original, 'He*llo World'); - assert.equal(query.value, 'HelloWorld'); - assert.equal(query.valueLowercase, 'HelloWorld'.toLowerCase()); + assert.equal(query.normalized, 'HelloWorld'); + assert.equal(query.normalizedLowercase, 'HelloWorld'.toLowerCase()); assert.equal(query.values?.length, 2); assert.equal(query.values?.[0].original, 'He*llo'); - assert.equal(query.values?.[0].value, 'Hello'); - assert.equal(query.values?.[0].valueLowercase, 'Hello'.toLowerCase()); + assert.equal(query.values?.[0].normalized, 'Hello'); + assert.equal(query.values?.[0].normalizedLowercase, 'Hello'.toLowerCase()); assert.equal(query.values?.[1].original, 'World'); - assert.equal(query.values?.[1].value, 'World'); - assert.equal(query.values?.[1].valueLowercase, 'World'.toLowerCase()); + assert.equal(query.values?.[1].normalized, 'World'); + assert.equal(query.values?.[1].normalizedLowercase, 'World'.toLowerCase()); // with spaces that are empty query = scorer.prepareQuery(' Hello World '); assert.equal(query.original, ' Hello World '); assert.equal(query.originalLowercase, ' Hello World '.toLowerCase()); - assert.equal(query.value, 'HelloWorld'); - assert.equal(query.valueLowercase, 'HelloWorld'.toLowerCase()); + assert.equal(query.normalized, 'HelloWorld'); + assert.equal(query.normalizedLowercase, 'HelloWorld'.toLowerCase()); assert.equal(query.values?.length, 2); assert.equal(query.values?.[0].original, 'Hello'); assert.equal(query.values?.[0].originalLowercase, 'Hello'.toLowerCase()); - assert.equal(query.values?.[0].value, 'Hello'); - assert.equal(query.values?.[0].valueLowercase, 'Hello'.toLowerCase()); + assert.equal(query.values?.[0].normalized, 'Hello'); + assert.equal(query.values?.[0].normalizedLowercase, 'Hello'.toLowerCase()); assert.equal(query.values?.[1].original, 'World'); assert.equal(query.values?.[1].originalLowercase, 'World'.toLowerCase()); - assert.equal(query.values?.[1].value, 'World'); - assert.equal(query.values?.[1].valueLowercase, 'World'.toLowerCase()); + assert.equal(query.values?.[1].normalized, 'World'); + assert.equal(query.values?.[1].normalizedLowercase, 'World'.toLowerCase()); + + // Path related + if (isWindows) { + assert.equal(scorer.prepareQuery('C:\\some\\path').pathNormalized, 'C:\\some\\path'); + assert.equal(scorer.prepareQuery('C:\\some\\path').normalized, 'C:\\some\\path'); + assert.equal(scorer.prepareQuery('C:\\some\\path').containsPathSeparator, true); + assert.equal(scorer.prepareQuery('C:/some/path').pathNormalized, 'C:\\some\\path'); + assert.equal(scorer.prepareQuery('C:/some/path').normalized, 'C:\\some\\path'); + assert.equal(scorer.prepareQuery('C:/some/path').containsPathSeparator, true); + } else { + assert.equal(scorer.prepareQuery('/some/path').pathNormalized, '/some/path'); + assert.equal(scorer.prepareQuery('/some/path').normalized, '/some/path'); + assert.equal(scorer.prepareQuery('/some/path').containsPathSeparator, true); + assert.equal(scorer.prepareQuery('\\some\\path').pathNormalized, '/some/path'); + assert.equal(scorer.prepareQuery('\\some\\path').normalized, '/some/path'); + assert.equal(scorer.prepareQuery('\\some\\path').containsPathSeparator, true); + } }); }); diff --git a/src/vs/workbench/browser/parts/editor/editorQuickAccess.ts b/src/vs/workbench/browser/parts/editor/editorQuickAccess.ts index a617ad1b11e..da1d8d3c31e 100644 --- a/src/vs/workbench/browser/parts/editor/editorQuickAccess.ts +++ b/src/vs/workbench/browser/parts/editor/editorQuickAccess.ts @@ -62,7 +62,7 @@ export abstract class BaseEditorQuickAccessProvider extends PickerQuickAccessPro // Filtering const filteredEditorEntries = this.doGetEditorPickItems().filter(entry => { - if (!query.value) { + if (!query.normalized) { return true; } @@ -79,7 +79,7 @@ export abstract class BaseEditorQuickAccessProvider extends PickerQuickAccessPro }); // Sorting - if (query.value) { + if (query.normalized) { const groups = this.editorGroupService.getGroups(GroupsOrder.GRID_APPEARANCE).map(group => group.id); filteredEditorEntries.sort((entryA, entryB) => { if (entryA.groupId !== entryB.groupId) { diff --git a/src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts b/src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts index 9ef538dd8db..53cfb5845eb 100644 --- a/src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts +++ b/src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts @@ -402,7 +402,7 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider this.createAnythingPick(editor, configuration)); } @@ -448,9 +448,9 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider(AnythingQuickAccessProvider.TYPING_SEARCH_DELAY)); + private readonly fileQueryDelayer = this._register(new ThrottledDelayer(AnythingQuickAccessProvider.TYPING_SEARCH_DELAY)); - private fileQueryBuilder = this.instantiationService.createInstance(QueryBuilder); + private readonly fileQueryBuilder = this.instantiationService.createInstance(QueryBuilder); private createFileQueryCache(): FileQueryCacheState { return new FileQueryCacheState( @@ -462,7 +462,7 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider, token: CancellationToken): Promise> { - if (!query.value) { + if (!query.normalized) { return []; } @@ -692,9 +692,9 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider> { const configuration = this.configuration; if ( - !query.value || // we need a value for search for - !configuration.includeSymbols || // we need to enable symbols in search - this.pickState.lastRange // a range is an indicator for just searching for files + !query.normalized || // we need a value for search for + !configuration.includeSymbols || // we need to enable symbols in search + this.pickState.lastRange // a range is an indicator for just searching for files ) { return []; } diff --git a/src/vs/workbench/services/search/node/fileSearch.ts b/src/vs/workbench/services/search/node/fileSearch.ts index 9a4a442d6ce..97fe8231b86 100644 --- a/src/vs/workbench/services/search/node/fileSearch.ts +++ b/src/vs/workbench/services/search/node/fileSearch.ts @@ -77,7 +77,7 @@ export class FileWalker { this.errors = []; if (this.filePattern) { - this.normalizedFilePatternLowercase = prepareQuery(this.filePattern).valueLowercase; + this.normalizedFilePatternLowercase = prepareQuery(this.filePattern).normalizedLowercase; } this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern); diff --git a/src/vs/workbench/services/search/node/rawSearchService.ts b/src/vs/workbench/services/search/node/rawSearchService.ts index a34a8e5be47..978ef077c60 100644 --- a/src/vs/workbench/services/search/node/rawSearchService.ts +++ b/src/vs/workbench/services/search/node/rawSearchService.ts @@ -312,7 +312,7 @@ export class SearchService implements IRawSearchService { // Pattern match on results const results: IRawFileMatch[] = []; - const normalizedSearchValueLowercase = prepareQuery(searchValue).valueLowercase; + const normalizedSearchValueLowercase = prepareQuery(searchValue).normalizedLowercase; for (const entry of cachedEntries) { // Check if this entry is a match for the search value -- GitLab