diff --git a/.vscode/launch.json b/.vscode/launch.json index 4b318401ae2050b50cdb4c6ddfc36b36ce981377..a3a72ea7e09d5f77ff70ab818f3009ec690c1006 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -63,6 +63,16 @@ "${workspaceRoot}/out/**/*.js" ] }, + { + "type": "node", + "request": "attach", + "name": "Attach to Search process", + "port": 7890, + "sourceMaps": true, + "outFiles": [ + "${workspaceRoot}/out/**/*.js" + ] + }, { "type": "node", "request": "attach", diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index efc4b4777051d5ba7e287d9257635ae2bc2fad10..86ce450ca919b9042ff8e3d17c53afde98a15345 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -410,9 +410,9 @@ "resolved": "https://registry.npmjs.org/vscode-debugprotocol/-/vscode-debugprotocol-1.17.0.tgz" }, "vscode-ripgrep": { - "version": "0.0.8", - "from": "vscode-ripgrep@0.0.8", - "resolved": "https://registry.npmjs.org/vscode-ripgrep/-/vscode-ripgrep-0.0.8.tgz" + "version": "0.0.11", + "from": "vscode-ripgrep@0.0.11", + "resolved": "https://registry.npmjs.org/vscode-ripgrep/-/vscode-ripgrep-0.0.11.tgz" }, "vscode-textmate": { "version": "3.1.1", diff --git a/package.json b/package.json index c55bc8d639fc99533927a3687cdb20a7535d3d8f..a7a83ed713319201e9c8872e04f782af25bf5596 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "semver": "4.3.6", "v8-profiler": "jrieken/v8-profiler#vscode", "vscode-debugprotocol": "1.17.0", - "vscode-ripgrep": "0.0.8", + "vscode-ripgrep": "0.0.11", "vscode-textmate": "^3.1.1", "winreg": "1.2.0", "xterm": "Tyriar/xterm.js#vscode-release/1.11", diff --git a/src/typings/vscode-ripgrep.d.ts b/src/typings/vscode-ripgrep.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c5c89c3ca89c393d1b0d72d1db361d0f61ede32 --- /dev/null +++ b/src/typings/vscode-ripgrep.d.ts @@ -0,0 +1,3 @@ +declare module 'vscode-ripgrep' { + export const rgPath: string; +} diff --git a/src/vs/platform/search/common/search.ts b/src/vs/platform/search/common/search.ts index a55a139eb62764a49f88cc14a182a184b8c8c381..e9fb3a1bab95c015d643369bf3b207b106b9967a 100644 --- a/src/vs/platform/search/common/search.ts +++ b/src/vs/platform/search/common/search.ts @@ -35,6 +35,7 @@ export interface IQueryOptions { sortByScore?: boolean; cacheKey?: string; fileEncoding?: string; + useRipgrep?: boolean; } export interface ISearchQuery extends IQueryOptions { @@ -127,6 +128,7 @@ export class LineMatch implements ILineMatch { export interface ISearchConfiguration extends IFilesConfiguration { search: { exclude: IExpression; + useRipgrep: boolean; }; } diff --git a/src/vs/workbench/parts/search/browser/search.contribution.ts b/src/vs/workbench/parts/search/browser/search.contribution.ts index d7aab6c80909aafe2d1a6f8f2a59ea6a357b5dbc..e0bf10418a8bec6a7e970a6485cb720b20e8bc4c 100644 --- a/src/vs/workbench/parts/search/browser/search.contribution.ts +++ b/src/vs/workbench/parts/search/browser/search.contribution.ts @@ -204,6 +204,11 @@ configurationRegistry.registerConfiguration({ ] } }, + 'search.useRipgrep': { + 'type': 'boolean', + 'description': nls.localize('useRipgrep', "Controls whether to use ripgrep in text search"), + 'default': false + }, 'search.quickOpen.includeSymbols': { 'type': 'boolean', 'description': nls.localize('search.quickOpen.includeSymbols', "Configure to include results from a global symbol search in the file results for Quick Open."), diff --git a/src/vs/workbench/parts/search/browser/searchViewlet.ts b/src/vs/workbench/parts/search/browser/searchViewlet.ts index 3d6e3cc64d846783bce091c6d58d63a67b4be694..805f6dd57eb952a305989858acc845e90caee23a 100644 --- a/src/vs/workbench/parts/search/browser/searchViewlet.ts +++ b/src/vs/workbench/parts/search/browser/searchViewlet.ts @@ -990,9 +990,12 @@ export class SearchViewlet extends Viewlet { // Progress total is 100.0% for more progress bar granularity let progressTotal = 1000; - let progressRunner = this.progressService.show(progressTotal); let progressWorked = 0; + let progressRunner = query.useRipgrep ? + this.progressService.show(/*infinite=*/true) : + this.progressService.show(progressTotal); + this.loading = true; this.searchWidget.searchInput.clearMessage(); this.showEmptyStage(); @@ -1017,7 +1020,7 @@ export class SearchViewlet extends Viewlet { isDone = true; // Complete up to 100% as needed - if (completed) { + if (completed && !query.useRipgrep) { progressRunner.worked(progressTotal - progressWorked); setTimeout(() => progressRunner.done(), 200); } else { @@ -1146,26 +1149,28 @@ export class SearchViewlet extends Viewlet { return; } - // Progress bar update - let fakeProgress = true; - if (total > 0 && worked > 0) { - let ratio = Math.round((worked / total) * progressTotal); - if (ratio > progressWorked) { // never show less progress than what we have already - progressRunner.worked(ratio - progressWorked); - progressWorked = ratio; - fakeProgress = false; + if (!query.useRipgrep) { + // Progress bar update + let fakeProgress = true; + if (total > 0 && worked > 0) { + let ratio = Math.round((worked / total) * progressTotal); + if (ratio > progressWorked) { // never show less progress than what we have already + progressRunner.worked(ratio - progressWorked); + progressWorked = ratio; + fakeProgress = false; + } } - } - // Fake progress up to 90%, or when actual progress beats it - const fakeMax = 900; - const fakeMultiplier = 12; - if (fakeProgress && progressWorked < fakeMax) { - // Linearly decrease the rate of fake progress. - // 1 is the smallest allowed amount of progress. - const fakeAmt = Math.round((fakeMax - progressWorked) / fakeMax * fakeMultiplier) || 1; - progressWorked += fakeAmt; - progressRunner.worked(fakeAmt); + // Fake progress up to 90%, or when actual progress beats it + const fakeMax = 900; + const fakeMultiplier = 12; + if (fakeProgress && progressWorked < fakeMax) { + // Linearly decrease the rate of fake progress. + // 1 is the smallest allowed amount of progress. + const fakeAmt = Math.round((fakeMax - progressWorked) / fakeMax * fakeMultiplier) || 1; + progressWorked += fakeAmt; + progressRunner.worked(fakeAmt); + } } // Search result tree update @@ -1188,7 +1193,7 @@ export class SearchViewlet extends Viewlet { this.searchWidget.setReplaceAllActionState(false); // this.replaceService.disposeAllReplacePreviews(); - this.viewModel.search(query).done(onComplete, onError, onProgress); + this.viewModel.search(query).done(onComplete, onError, query.useRipgrep ? undefined : onProgress); } private updateSearchResultCount(): void { diff --git a/src/vs/workbench/parts/search/common/search.ts b/src/vs/workbench/parts/search/common/search.ts index 156663740132a53ef947398ed0ec43b0916a5166..9725f2d9294881dc9ef0b1770286a2a4287fed23 100644 --- a/src/vs/workbench/parts/search/common/search.ts +++ b/src/vs/workbench/parts/search/common/search.ts @@ -86,6 +86,7 @@ export interface IWorkbenchSearchConfiguration extends ISearchConfiguration { quickOpen: { includeSymbols: boolean; }, - exclude: glob.IExpression; + exclude: glob.IExpression, + useRipgrep: boolean }; } \ No newline at end of file diff --git a/src/vs/workbench/parts/search/common/searchQuery.ts b/src/vs/workbench/parts/search/common/searchQuery.ts index ac61c9adb1299a8becb14f5635c8d7d703c42b02..3e9a193f4f4ee2724375974da6040e3827eada10 100644 --- a/src/vs/workbench/parts/search/common/searchQuery.ts +++ b/src/vs/workbench/parts/search/common/searchQuery.ts @@ -42,7 +42,8 @@ export class QueryBuilder { sortByScore: options.sortByScore, cacheKey: options.cacheKey, fileEncoding: options.fileEncoding, - contentPattern: contentPattern + contentPattern: contentPattern, + useRipgrep: configuration.search.useRipgrep }; } } \ No newline at end of file diff --git a/src/vs/workbench/services/search/node/rawSearchService.ts b/src/vs/workbench/services/search/node/rawSearchService.ts index fec0087716b6b976c13f2f0e46571b83b8825f6d..96fa12ce8134d756db5e8ed2c1a1496938b48803 100644 --- a/src/vs/workbench/services/search/node/rawSearchService.ts +++ b/src/vs/workbench/services/search/node/rawSearchService.ts @@ -17,8 +17,9 @@ import objects = require('vs/base/common/objects'); import scorer = require('vs/base/common/scorer'); import strings = require('vs/base/common/strings'); import { PPromise, TPromise } from 'vs/base/common/winjs.base'; -import { MAX_FILE_SIZE } from 'vs/platform/files/common/files'; import { FileWalker, Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch'; +import { MAX_FILE_SIZE } from 'vs/platform/files/common/files'; +import { RipgrepEngine } from 'vs/workbench/services/search/node/ripgrepTextSearch'; import { Engine as TextSearchEngine } from 'vs/workbench/services/search/node/textSearch'; import { TextSearchWorkerProvider } from 'vs/workbench/services/search/node/textSearchWorkerProvider'; import { IRawSearchService, IRawSearch, IRawFileMatch, ISerializedFileMatch, ISerializedSearchProgressItem, ISerializedSearchComplete, ISearchEngine } from './search'; @@ -39,6 +40,37 @@ export class SearchService implements IRawSearchService { } public textSearch(config: IRawSearch): PPromise { + return config.useRipgrep ? + this.ripgrepTextSearch(config) : + this.legacyTextSearch(config); + } + + public ripgrepTextSearch(config: IRawSearch): PPromise { + config.maxFilesize = MAX_FILE_SIZE; + let engine = new RipgrepEngine(config); + + return new PPromise>((c, e, p) => { + // Use BatchedCollector to get new results to the frontend every 2s at least, until 50 results have been returned + const collector = new BatchedCollector(SearchService.BATCH_SIZE, p); + engine.search((match) => { + collector.addItem(match, match.numMatches); + }, (progress) => { + p(progress); + }, (error, stats) => { + collector.flush(); + + if (error) { + e(error); + } else { + c(stats); + } + }); + }, () => { + engine.cancel(); + }); + } + + public legacyTextSearch(config: IRawSearch): PPromise { if (!this.textSearchWorkerProvider) { this.textSearchWorkerProvider = new TextSearchWorkerProvider(); } @@ -399,6 +431,18 @@ class BatchedCollector { constructor(private maxBatchSize: number, private cb: (items: T | T[]) => void) { } + addItem(item: T, size: number): void { + if (!item) { + return; + } + + if (this.maxBatchSize > 0) { + this.addItemToBatch(item, size); + } else { + this.cb(item); + } + } + addItems(items: T[], size: number): void { if (!items) { return; @@ -411,9 +455,19 @@ class BatchedCollector { } } - private addItemsToBatch(items: T[], size: number): void { - this.batch = this.batch.concat(items); + private addItemToBatch(item: T, size: number): void { + this.batch.push(item); this.batchSize += size; + this.onUpdate(); + } + + private addItemsToBatch(item: T[], size: number): void { + this.batch = this.batch.concat(item); + this.batchSize += size; + this.onUpdate(); + } + + private onUpdate(): void { if (this.totalNumberCompleted < BatchedCollector.START_BATCH_AFTER_COUNT) { // Flush because we aren't batching yet this.flush(); diff --git a/src/vs/workbench/services/search/node/ripgrepTextSearch.ts b/src/vs/workbench/services/search/node/ripgrepTextSearch.ts new file mode 100644 index 0000000000000000000000000000000000000000..c213bd465531e52f4f4356feeb84eb7cdee175c1 --- /dev/null +++ b/src/vs/workbench/services/search/node/ripgrepTextSearch.ts @@ -0,0 +1,337 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { EventEmitter } from 'events'; + +import * as cp from 'child_process'; +import { rgPath } from 'vscode-ripgrep'; + +import * as strings from 'vs/base/common/strings'; +import * as glob from 'vs/base/common/glob'; +import { ILineMatch, IProgress } from 'vs/platform/search/common/search'; + +import { ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine } from './search'; + +export class RipgrepEngine implements ISearchEngine { + private isDone = false; + private rgProc: cp.ChildProcess; + private postProcessExclusions: glob.SiblingClause[]; + + private ripgrepParser: RipgrepParser; + + constructor(private config: IRawSearch) { + } + + cancel(): void { + this.isDone = true; + this.ripgrepParser.cancel(); + this.rgProc.kill(); + } + + // TODO@Rob - make promise-based once the old search is gone, and I don't need them to have matching interfaces anymore + search(onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { + if (this.config.rootFolders.length) { + this.searchFolder(this.config.rootFolders[0], onResult, onProgress, done); + } else { + done(null, { + limitHit: false, + stats: null + }); + } + } + + private searchFolder(rootFolder: string, onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { + const rgArgs = getRgArgs(this.config, rootFolder); + this.postProcessExclusions = rgArgs.siblingClauses; + + // console.log(`rg ${rgArgs.join(' ')}, cwd: ${rootFolder}`); + this.rgProc = cp.spawn(rgPath, rgArgs.args, { cwd: rootFolder }); + + this.ripgrepParser = new RipgrepParser(this.config.maxResults); + this.ripgrepParser.on('result', onResult); + this.ripgrepParser.on('hitLimit', () => { + this.cancel(); + done(null, { + limitHit: true, + stats: null + }); + }); + + this.rgProc.stdout.on('data', data => { + this.ripgrepParser.handleData(data); + }); + + this.rgProc.stderr.on('data', data => { + // TODO@rob remove console.logs + console.log('stderr:'); + console.log(data.toString()); + }); + + this.rgProc.on('close', code => { + this.rgProc = null; + // console.log(`closed with ${code}`); + + if (!this.isDone) { + this.isDone = true; + done(null, { + limitHit: false, + stats: null + }); + } + }); + } +} + +export class RipgrepParser extends EventEmitter { + private static RESULT_REGEX = /^\u001b\[m(\d+)\u001b\[m:(.*)$/; + private static FILE_REGEX = /^\u001b\[m(.+)\u001b\[m$/; + + private static MATCH_START_MARKER = '\u001b[m\u001b[31m'; + private static MATCH_END_MARKER = '\u001b[m'; + + private fileMatch: FileMatch; + private remainder: string; + private isDone: boolean; + + private numResults = 0; + + constructor(private maxResults: number) { + super(); + } + + public cancel(): void { + this.isDone = true; + } + + public handleData(data: string | Buffer): void { + // If the previous data chunk didn't end in a newline, append it to this chunk + const dataStr = this.remainder ? + this.remainder + data.toString() : + data.toString(); + + const dataLines: string[] = dataStr.split(/\r\n|\n/); + this.remainder = dataLines[dataLines.length - 1] ? dataLines.pop() : null; + + for (let l = 0; l < dataLines.length; l++) { + const outputLine = dataLines[l].trim(); + if (this.isDone) { + break; + } + + let r: RegExpMatchArray; + if (!outputLine) { + if (this.fileMatch) { + this.onResult(); + } + } else if (r = outputLine.match(RipgrepParser.RESULT_REGEX)) { + // Line is a result - add to collected results for the current file path + this.handleMatchLine(outputLine, parseInt(r[1]) - 1, r[2]); + } else if (r = outputLine.match(RipgrepParser.FILE_REGEX)) { + // Line is a file path - send all collected results for the previous file path + if (this.fileMatch) { + // TODO@Rob Check fileMatch against other exclude globs + this.onResult(); + } + + this.fileMatch = new FileMatch(r[1]); + } else { + // Line is malformed + } + } + } + + private handleMatchLine(outputLine: string, lineNum: number, text: string): void { + const lineMatch = new LineMatch(text, lineNum); + this.fileMatch.addMatch(lineMatch); + + let lastMatchEndPos = 0; + let matchTextStartPos = -1; + + // Track positions with color codes subtracted - offsets in the final text preview result + let matchTextStartRealIdx = -1; + let textRealIdx = 0; + + const realTextParts: string[] = []; + + // todo@Rob Consider just rewriting with a regex. I think perf will be fine. + for (let i = 0; i < text.length - (RipgrepParser.MATCH_END_MARKER.length - 1);) { + if (text.substr(i, RipgrepParser.MATCH_START_MARKER.length) === RipgrepParser.MATCH_START_MARKER) { + // Match start + const chunk = text.slice(lastMatchEndPos, i); + realTextParts.push(chunk); + i += RipgrepParser.MATCH_START_MARKER.length; + matchTextStartPos = i; + matchTextStartRealIdx = textRealIdx; + } else if (text.substr(i, RipgrepParser.MATCH_END_MARKER.length) === RipgrepParser.MATCH_END_MARKER) { + // Match end + const chunk = text.slice(matchTextStartPos, i); + realTextParts.push(chunk); + lineMatch.addMatch(matchTextStartRealIdx, textRealIdx - matchTextStartRealIdx); + matchTextStartPos = -1; + matchTextStartRealIdx = -1; + i += RipgrepParser.MATCH_END_MARKER.length; + lastMatchEndPos = i; + this.numResults++; + + // Check hit maxResults limit + if (this.numResults >= this.maxResults) { + // Replace preview with what we have so far, TODO@Rob + lineMatch.preview = realTextParts.join(''); + this.cancel(); + this.onResult(); + this.emit('hitLimit'); + } + } else { + i++; + textRealIdx++; + } + } + + const chunk = text.slice(lastMatchEndPos); + realTextParts.push(chunk); + + // Replace preview with version without color codes + const preview = realTextParts.join(''); + lineMatch.preview = preview; + } + + private onResult(): void { + this.emit('result', this.fileMatch.serialize()); + this.fileMatch = null; + } +} + +export class FileMatch implements ISerializedFileMatch { + path: string; + lineMatches: LineMatch[]; + + constructor(path: string) { + this.path = path; + this.lineMatches = []; + } + + addMatch(lineMatch: LineMatch): void { + this.lineMatches.push(lineMatch); + } + + isEmpty(): boolean { + return this.lineMatches.length === 0; + } + + serialize(): ISerializedFileMatch { + let lineMatches: ILineMatch[] = []; + let numMatches = 0; + + for (let i = 0; i < this.lineMatches.length; i++) { + numMatches += this.lineMatches[i].offsetAndLengths.length; + lineMatches.push(this.lineMatches[i].serialize()); + } + + return { + path: this.path, + lineMatches, + numMatches + }; + } +} + +export class LineMatch implements ILineMatch { + preview: string; + lineNumber: number; + offsetAndLengths: number[][]; + + constructor(preview: string, lineNumber: number) { + this.preview = preview.replace(/(\r|\n)*$/, ''); + this.lineNumber = lineNumber; + this.offsetAndLengths = []; + } + + getText(): string { + return this.preview; + } + + getLineNumber(): number { + return this.lineNumber; + } + + addMatch(offset: number, length: number): void { + this.offsetAndLengths.push([offset, length]); + } + + serialize(): ILineMatch { + const result = { + preview: this.preview, + lineNumber: this.lineNumber, + offsetAndLengths: this.offsetAndLengths + }; + + return result; + } +} + +function globExprsToRgGlobs(patterns: glob.IExpression): { globArgs: string[], siblingClauses: glob.SiblingClause[] } { + const globArgs: string[] = []; + const siblingClauses: glob.SiblingClause[] = []; + Object.keys(patterns) + .forEach(key => { + const value = patterns[key]; + if (typeof value === 'boolean' && value) { + // globs added to ripgrep don't match from the root by default, so add a / + if (key.charAt(0) !== '*') { + key = '/' + key; + } + + globArgs.push(key); + } else if (value && value.when) { + siblingClauses.push(value); + } + }); + + return { globArgs, siblingClauses }; +} + +function getRgArgs(config: IRawSearch, rootFolder: string): { args: string[], siblingClauses: glob.SiblingClause[] } { + const args = ['--heading', '--line-number', '--color', 'ansi', '--colors', 'path:none', '--colors', 'line:none', '--colors', 'match:fg:red', '--colors', 'match:style:nobold']; + args.push(config.contentPattern.isCaseSensitive ? '--case-sensitive' : '--ignore-case'); + + if (config.includePattern) { + // I don't think includePattern can have siblingClauses + globExprsToRgGlobs(config.includePattern).globArgs.forEach(globArg => { + args.push('-g', globArg); + }); + } + + let siblingClauses: glob.SiblingClause[] = []; + if (config.excludePattern) { + const rgGlobs = globExprsToRgGlobs(config.excludePattern); + rgGlobs.globArgs + .forEach(rgGlob => args.push('-g', `!${rgGlob}`)); + siblingClauses = rgGlobs.siblingClauses; + } + + if (config.maxFilesize) { + args.push('--max-filesize', config.maxFilesize + ''); + } + + if (config.contentPattern.isRegExp) { + if (config.contentPattern.isWordMatch) { + args.push('--word-regexp'); + } + + args.push('--regexp', config.contentPattern.pattern); + } else { + if (config.contentPattern.isWordMatch) { + args.push('--word-regexp', '--regexp', strings.escapeRegExpCharacters(config.contentPattern.pattern)); + } else { + args.push('--fixed-strings', config.contentPattern.pattern); + } + } + + // Folder to search + args.push('--', rootFolder); + + return { args, siblingClauses }; +} diff --git a/src/vs/workbench/services/search/node/search.ts b/src/vs/workbench/services/search/node/search.ts index ee26f58147590becbebee4aba7d61f61614e0e9e..7552d44c8dea799df3570803da19ac4648367d48 100644 --- a/src/vs/workbench/services/search/node/search.ts +++ b/src/vs/workbench/services/search/node/search.ts @@ -21,6 +21,7 @@ export interface IRawSearch { cacheKey?: string; maxFilesize?: number; fileEncoding?: string; + useRipgrep?: boolean; } export interface IRawSearchService { diff --git a/src/vs/workbench/services/search/node/searchService.ts b/src/vs/workbench/services/search/node/searchService.ts index 9ea10c7de733701a6a7d5dd87b8be6ab2dc40d9f..1bbed9470443a61b0840f79277f9b617d30f662a 100644 --- a/src/vs/workbench/services/search/node/searchService.ts +++ b/src/vs/workbench/services/search/node/searchService.ts @@ -234,7 +234,8 @@ export class DiskSearch { includePattern: query.includePattern, maxResults: query.maxResults, sortByScore: query.sortByScore, - cacheKey: query.cacheKey + cacheKey: query.cacheKey, + useRipgrep: query.useRipgrep }; if (query.type === QueryType.Text) { diff --git a/src/vs/workbench/services/search/test/node/textSearch.integrationTest.ts b/src/vs/workbench/services/search/test/node/textSearch.integrationTest.ts index 342fa76c23c5cea5957560118ffb76ed8f0192c5..981013ea9bf779530757ca7476a1300b8dd182e1 100644 --- a/src/vs/workbench/services/search/test/node/textSearch.integrationTest.ts +++ b/src/vs/workbench/services/search/test/node/textSearch.integrationTest.ts @@ -8,28 +8,15 @@ import path = require('path'); import assert = require('assert'); -import { LineMatch } from 'vs/platform/search/common/search'; - +import { TPromise } from 'vs/base/common/winjs.base'; import { FileWalker } from 'vs/workbench/services/search/node/fileSearch'; -import { ISerializedFileMatch } from 'vs/workbench/services/search/node/search'; +import { ISerializedFileMatch, IRawSearch } from 'vs/workbench/services/search/node/search'; import { Engine as TextSearchEngine } from 'vs/workbench/services/search/node/textSearch'; +import { RipgrepEngine } from 'vs/workbench/services/search/node/ripgrepTextSearch'; import { TextSearchWorkerProvider } from 'vs/workbench/services/search/node/textSearchWorkerProvider'; function countAll(matches: ISerializedFileMatch[]): number { - return matches.reduce((acc, m) => acc + count(m.lineMatches), 0); -} - -function count(lineMatches: LineMatch[]): number { - let count = 0; - if (lineMatches) { - for (let i = 0; i < lineMatches.length; i++) { - let line = lineMatches[i]; - let wordMatches = line.offsetAndLengths; - count += wordMatches.length; - } - } - - return count; + return matches.reduce((acc, m) => acc + m.numMatches, 0); } function rootfolders() { @@ -37,114 +24,103 @@ function rootfolders() { } const textSearchWorkerProvider = new TextSearchWorkerProvider(); -suite('Search-integration', () => { - test('Text: GameOfLife', function (done: () => void) { - let c = 0; - let config = { - rootFolders: rootfolders(), - filePattern: '*.js', - contentPattern: { pattern: 'GameOfLife', modifiers: 'i' } - }; +function doLegacySearchTest(config: IRawSearch, expectedResultCount: number | Function): TPromise { + return new TPromise(resolve => { let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); + let c = 0; engine.search((result) => { if (result) { c += countAll(result); } }, () => { }, (error) => { assert.ok(!error); - assert.equal(c, 4); - done(); + if (typeof expectedResultCount === 'function') { + assert(expectedResultCount(c)); + } else { + assert.equal(c, expectedResultCount); + } + resolve(undefined); }); }); +} - test('Text: GameOfLife (RegExp)', function (done: () => void) { - let c = 0; - let config = { - rootFolders: rootfolders(), - filePattern: '*.js', - contentPattern: { pattern: 'Game.?fL\\w?fe', isRegExp: true } - }; - - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); +function doRipgrepSearchTest(config: IRawSearch, expectedResultCount: number): TPromise { + return new TPromise(resolve => { + let engine = new RipgrepEngine(config); + let c = 0; engine.search((result) => { if (result) { - c += countAll(result); + c += result.numMatches; } }, () => { }, (error) => { assert.ok(!error); - assert.equal(c, 4); - done(); + assert.equal(c, expectedResultCount); + resolve(undefined); }); }); +} + +function doSearchTest(config: IRawSearch, expectedResultCount: number, done) { + return doLegacySearchTest(config, expectedResultCount) + // .then(() => doRipgrepSearchTest(config, expectedResultCount)) + .then(done, done); +} + +suite('Search-integration', () => { + test('Text: GameOfLife', function (done: () => void) { + let config = { + rootFolders: rootfolders(), + filePattern: '*.js', + contentPattern: { pattern: 'GameOfLife', modifiers: 'i' }, + }; + + doSearchTest(config, 4, done); + }); + + test('Text: GameOfLife (RegExp)', function (done: () => void) { + let config = { + rootFolders: rootfolders(), + filePattern: '*.js', + contentPattern: { pattern: 'Game.?fL\\w?fe', isRegExp: true } + }; + + doSearchTest(config, 4, done); + }); test('Text: GameOfLife (Word Match, Case Sensitive)', function (done: () => void) { - let c = 0; let config = { rootFolders: rootfolders(), filePattern: '*.js', contentPattern: { pattern: 'GameOfLife', isWordMatch: true, isCaseSensitive: true } }; - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); - - engine.search((result) => { - if (result) { - c += countAll(result); - } - }, () => { }, (error) => { - assert.ok(!error); - assert.equal(c, 4); - done(); - }); + doSearchTest(config, 4, done); }); test('Text: Helvetica (UTF 16)', function (done: () => void) { - let c = 0; let config = { rootFolders: rootfolders(), filePattern: '*.css', contentPattern: { pattern: 'Helvetica', modifiers: 'i' } }; - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); - - engine.search((result) => { - if (result) { - c += countAll(result); - } - }, () => { }, (error) => { - assert.ok(!error); - assert.equal(c, 3); - done(); - }); + doSearchTest(config, 3, done); }); test('Text: e', function (done: () => void) { - let c = 0; let config = { rootFolders: rootfolders(), filePattern: '*.*', contentPattern: { pattern: 'e', modifiers: 'i' } }; - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); - - engine.search((result) => { - if (result) { - c += countAll(result); - } - }, (result) => { }, (error) => { - assert.ok(!error); - assert.equal(c, 776); - done(); - }); + doSearchTest(config, 776, done); }); test('Text: e (with excludes)', function (done: () => void) { - let c = 0; let config: any = { rootFolders: rootfolders(), filePattern: '*.*', @@ -152,21 +128,10 @@ suite('Search-integration', () => { excludePattern: { '**/examples': true } }; - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); - - engine.search((result) => { - if (result) { - c += countAll(result); - } - }, (result) => { }, (error) => { - assert.ok(!error); - assert.equal(c, 394); - done(); - }); + doSearchTest(config, 394, done); }); test('Text: e (with includes)', function (done: () => void) { - let c = 0; let config: any = { rootFolders: rootfolders(), filePattern: '*.*', @@ -174,21 +139,10 @@ suite('Search-integration', () => { includePattern: { '**/examples/**': true } }; - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); - - engine.search((result) => { - if (result) { - c += countAll(result); - } - }, (result) => { }, (error) => { - assert.ok(!error); - assert.equal(c, 382); - done(); - }); + doSearchTest(config, 382, done); }); test('Text: e (with includes and exclude)', function (done: () => void) { - let c = 0; let config: any = { rootFolders: rootfolders(), filePattern: '*.*', @@ -197,62 +151,32 @@ suite('Search-integration', () => { excludePattern: { '**/examples/small.js': true } }; - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); - - engine.search((result) => { - if (result) { - c += countAll(result); - } - }, (result) => { }, (error) => { - assert.ok(!error); - assert.equal(c, 361); - done(); - }); + doSearchTest(config, 361, done); }); test('Text: a (capped)', function (done: () => void) { - let c = 0; + const maxResults = 520; let config = { rootFolders: rootfolders(), filePattern: '*.*', contentPattern: { pattern: 'a', modifiers: 'i' }, - maxResults: 520 + maxResults }; - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); - - engine.search((result) => { - if (result) { - c += countAll(result); - } - }, (result) => { }, (error) => { - assert.ok(!error); - - // Search can go over the maxResults because it doesn't trim the results from its worker processes to the exact max size. - // But the worst-case scenario should be 2*max-1 - assert.ok(c < 520 * 2); - done(); - }); + // (Legacy) search can go over the maxResults because it doesn't trim the results from its worker processes to the exact max size. + // But the worst-case scenario should be 2*max-1 + return doLegacySearchTest(config, count => count < maxResults * 2) + .then(() => doRipgrepSearchTest(config, maxResults)) + .then(done, done); }); test('Text: a (no results)', function (done: () => void) { - let c = 0; let config = { rootFolders: rootfolders(), filePattern: '*.*', contentPattern: { pattern: 'ahsogehtdas', modifiers: 'i' } }; - let engine = new TextSearchEngine(config, new FileWalker(config), textSearchWorkerProvider); - - engine.search((result) => { - if (result) { - c += countAll(result); - } - }, (result) => { }, (error) => { - assert.ok(!error); - assert.equal(c, 0); - done(); - }); + doSearchTest(config, 0, done); }); }); \ No newline at end of file