diff --git a/.vscode/launch.json b/.vscode/launch.json index 4b318401ae2050b50cdb4c6ddfc36b36ce981377..a3a72ea7e09d5f77ff70ab818f3009ec690c1006 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -63,6 +63,16 @@ "${workspaceRoot}/out/**/*.js" ] }, + { + "type": "node", + "request": "attach", + "name": "Attach to Search process", + "port": 7890, + "sourceMaps": true, + "outFiles": [ + "${workspaceRoot}/out/**/*.js" + ] + }, { "type": "node", "request": "attach", diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index efc4b4777051d5ba7e287d9257635ae2bc2fad10..475e123efd087200f50643f19bce98bf215d1455 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -410,9 +410,9 @@ "resolved": "https://registry.npmjs.org/vscode-debugprotocol/-/vscode-debugprotocol-1.17.0.tgz" }, "vscode-ripgrep": { - "version": "0.0.8", - "from": "vscode-ripgrep@0.0.8", - "resolved": "https://registry.npmjs.org/vscode-ripgrep/-/vscode-ripgrep-0.0.8.tgz" + "version": "0.0.10", + "from": "vscode-ripgrep@0.0.10", + "resolved": "https://registry.npmjs.org/vscode-ripgrep/-/vscode-ripgrep-0.0.10.tgz" }, "vscode-textmate": { "version": "3.1.1", diff --git a/package.json b/package.json index c55bc8d639fc99533927a3687cdb20a7535d3d8f..ffb2141a2b69dd97410b6b88cc676f67262a50b0 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "semver": "4.3.6", "v8-profiler": "jrieken/v8-profiler#vscode", "vscode-debugprotocol": "1.17.0", - "vscode-ripgrep": "0.0.8", + "vscode-ripgrep": "0.0.10", "vscode-textmate": "^3.1.1", "winreg": "1.2.0", "xterm": "Tyriar/xterm.js#vscode-release/1.11", diff --git a/src/typings/vscode-ripgrep.d.ts b/src/typings/vscode-ripgrep.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c5c89c3ca89c393d1b0d72d1db361d0f61ede32 --- /dev/null +++ b/src/typings/vscode-ripgrep.d.ts @@ -0,0 +1,3 @@ +declare module 'vscode-ripgrep' { + export const rgPath: string; +} diff --git a/src/vs/workbench/parts/search/browser/searchViewlet.ts b/src/vs/workbench/parts/search/browser/searchViewlet.ts index 3d6e3cc64d846783bce091c6d58d63a67b4be694..69107194720b566a000ef99a1e871d4417806298 100644 --- a/src/vs/workbench/parts/search/browser/searchViewlet.ts +++ b/src/vs/workbench/parts/search/browser/searchViewlet.ts @@ -35,7 +35,7 @@ import { Viewlet } from 'vs/workbench/browser/viewlet'; import { Match, FileMatch, SearchModel, FileMatchOrMatch, IChangeEvent, ISearchWorkbenchService } from 'vs/workbench/parts/search/common/searchModel'; import { QueryBuilder } from 'vs/workbench/parts/search/common/searchQuery'; import { MessageType, InputBox } from 'vs/base/browser/ui/inputbox/inputBox'; -import { getExcludes, ISearchProgressItem, ISearchComplete, ISearchQuery, IQueryOptions, ISearchConfiguration } from 'vs/platform/search/common/search'; +import { getExcludes, ISearchComplete, ISearchQuery, IQueryOptions, ISearchConfiguration } from 'vs/platform/search/common/search'; import { IWorkbenchEditorService } from 'vs/workbench/services/editor/common/editorService'; import { IStorageService, StorageScope } from 'vs/platform/storage/common/storage'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; @@ -988,10 +988,7 @@ export class SearchViewlet extends Viewlet { private onQueryTriggered(query: ISearchQuery, excludePattern: string, includePattern: string): void { this.viewModel.cancelSearch(); - // Progress total is 100.0% for more progress bar granularity - let progressTotal = 1000; - let progressRunner = this.progressService.show(progressTotal); - let progressWorked = 0; + let progressRunner = this.progressService.show(/*infinite=*/true); this.loading = true; this.searchWidget.searchInput.clearMessage(); @@ -1015,14 +1012,7 @@ export class SearchViewlet extends Viewlet { let isDone = false; let onComplete = (completed?: ISearchComplete) => { isDone = true; - - // Complete up to 100% as needed - if (completed) { - progressRunner.worked(progressTotal - progressWorked); - setTimeout(() => progressRunner.done(), 200); - } else { - progressRunner.done(); - } + progressRunner.done(); this.onSearchResultsChanged().then(() => autoExpand(true)); this.viewModel.replaceString = this.searchWidget.getReplaceValue(); @@ -1126,18 +1116,7 @@ export class SearchViewlet extends Viewlet { } }; - let total: number = 0; - let worked: number = 0; let visibleMatches = 0; - let onProgress = (p: ISearchProgressItem) => { - // Progress - if (p.total) { - total = p.total; - } - if (p.worked) { - worked = p.worked; - } - }; // Handle UI updates in an interval to show frequent progress and results let uiRefreshHandle = setInterval(() => { @@ -1146,28 +1125,6 @@ export class SearchViewlet extends Viewlet { return; } - // Progress bar update - let fakeProgress = true; - if (total > 0 && worked > 0) { - let ratio = Math.round((worked / total) * progressTotal); - if (ratio > progressWorked) { // never show less progress than what we have already - progressRunner.worked(ratio - progressWorked); - progressWorked = ratio; - fakeProgress = false; - } - } - - // Fake progress up to 90%, or when actual progress beats it - const fakeMax = 900; - const fakeMultiplier = 12; - if (fakeProgress && progressWorked < fakeMax) { - // Linearly decrease the rate of fake progress. - // 1 is the smallest allowed amount of progress. - const fakeAmt = Math.round((fakeMax - progressWorked) / fakeMax * fakeMultiplier) || 1; - progressWorked += fakeAmt; - progressRunner.worked(fakeAmt); - } - // Search result tree update const fileCount = this.viewModel.searchResult.fileCount(); if (visibleMatches !== fileCount) { @@ -1188,7 +1145,7 @@ export class SearchViewlet extends Viewlet { this.searchWidget.setReplaceAllActionState(false); // this.replaceService.disposeAllReplacePreviews(); - this.viewModel.search(query).done(onComplete, onError, onProgress); + this.viewModel.search(query).done(onComplete, onError); } private updateSearchResultCount(): void { diff --git a/src/vs/workbench/services/search/node/rawSearchService.ts b/src/vs/workbench/services/search/node/rawSearchService.ts index fec0087716b6b976c13f2f0e46571b83b8825f6d..fef90d139420b6dd9446a1328512518876031824 100644 --- a/src/vs/workbench/services/search/node/rawSearchService.ts +++ b/src/vs/workbench/services/search/node/rawSearchService.ts @@ -17,10 +17,8 @@ import objects = require('vs/base/common/objects'); import scorer = require('vs/base/common/scorer'); import strings = require('vs/base/common/strings'); import { PPromise, TPromise } from 'vs/base/common/winjs.base'; -import { MAX_FILE_SIZE } from 'vs/platform/files/common/files'; -import { FileWalker, Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch'; -import { Engine as TextSearchEngine } from 'vs/workbench/services/search/node/textSearch'; -import { TextSearchWorkerProvider } from 'vs/workbench/services/search/node/textSearchWorkerProvider'; +import { Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch'; +import { RipgrepEngine } from 'vs/workbench/services/search/node/ripgrepTextSearch'; import { IRawSearchService, IRawSearch, IRawFileMatch, ISerializedFileMatch, ISerializedSearchProgressItem, ISerializedSearchComplete, ISearchEngine } from './search'; import { ICachedSearchStats, IProgress } from 'vs/platform/search/common/search'; @@ -32,29 +30,12 @@ export class SearchService implements IRawSearchService { private caches: { [cacheKey: string]: Cache; } = Object.create(null); - private textSearchWorkerProvider: TextSearchWorkerProvider; - public fileSearch(config: IRawSearch): PPromise { return this.doFileSearch(FileSearchEngine, config, SearchService.BATCH_SIZE); } public textSearch(config: IRawSearch): PPromise { - if (!this.textSearchWorkerProvider) { - this.textSearchWorkerProvider = new TextSearchWorkerProvider(); - } - - let engine = new TextSearchEngine( - config, - new FileWalker({ - rootFolders: config.rootFolders, - extraFiles: config.extraFiles, - includePattern: config.includePattern, - excludePattern: config.excludePattern, - filePattern: config.filePattern, - maxFilesize: MAX_FILE_SIZE - }), - this.textSearchWorkerProvider); - + let engine = new RipgrepEngine(config); return this.doTextSearch(engine, SearchService.BATCH_SIZE); } @@ -279,7 +260,7 @@ export class SearchService implements IRawSearchService { }); } - private doTextSearch(engine: TextSearchEngine, batchSize: number): PPromise> { + private doTextSearch(engine: RipgrepEngine, batchSize: number): PPromise> { return new PPromise>((c, e, p) => { // Use BatchedCollector to get new results to the frontend every 2s at least, until 50 results have been returned const collector = new BatchedCollector(batchSize, p); diff --git a/src/vs/workbench/services/search/node/ripgrepTextSearch.ts b/src/vs/workbench/services/search/node/ripgrepTextSearch.ts new file mode 100644 index 0000000000000000000000000000000000000000..89ec08ce9b7194a4f52b4e7359b668beec00987b --- /dev/null +++ b/src/vs/workbench/services/search/node/ripgrepTextSearch.ts @@ -0,0 +1,284 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import * as cp from 'child_process'; +import * as path from 'path'; +import { rgPath } from 'vscode-ripgrep'; + +import * as strings from 'vs/base/common/strings'; +import * as glob from 'vs/base/common/glob'; +import { ILineMatch, IProgress } from 'vs/platform/search/common/search'; + +import { ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine } from './search'; + +export class RipgrepEngine implements ISearchEngine { + private static RESULT_REGEX = /^\u001b\[m(\d+)\u001b\[m:(.*)$/; + private static FILE_REGEX = /^\u001b\[m(.+)\u001b\[m$/; + + private static MATCH_START_MARKER = '\u001b[m\u001b[31m'; + private static MATCH_END_MARKER = '\u001b[m'; + + private config: IRawSearch; + private isDone = false; + private rgProc: cp.ChildProcess; + private postProcessExclusions: glob.SiblingClause[] = []; + + private numResults = 0; + + constructor(config: IRawSearch) { + this.config = config; + } + + cancel(): void { + this.isDone = true; + + this.rgProc.kill(); + } + + search(onResult: (match: ISerializedFileMatch[]) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { + this.searchNextFolder(onResult, onProgress, done); + } + + private searchNextFolder(onResult: (match: ISerializedFileMatch[]) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { + if (this.config.rootFolders.length) { + // TODO search all root folders + this.searchFolder(this.config.rootFolders.shift(), onResult, onProgress, done); + } + } + + private searchFolder(rootFolder: string, onResult: (match: ISerializedFileMatch[]) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { + const rgArgs = this.getRgArgs(); + console.log(`rg ${rgArgs.join(' ')}, cwd: ${rootFolder}`); + this.rgProc = cp.spawn(rgPath, rgArgs, { cwd: rootFolder }); + + let fileMatch: FileMatch; + let remainder: string; + this.rgProc.stdout.on('data', data => { + // If the previous data chunk didn't end in a newline, append it to this chunk + const dataStr = remainder ? + remainder + data.toString() : + data.toString(); + + const dataLines: string[] = dataStr.split('\n'); + remainder = dataLines.pop(); + + for (let l = 0; l < dataLines.length; l++) { + const outputLine = dataLines[l]; + if (this.isDone) { + break; + } + + let r = outputLine.match(RipgrepEngine.RESULT_REGEX); + if (r) { + // Line is a result - add to collected results for the current file path + const line = parseInt(r[1]) - 1; + const text = r[2]; + + const lineMatch = new LineMatch(text, line); + fileMatch.addMatch(lineMatch); + + let lastMatchEndPos = 0; + let matchTextStartPos = -1; + + // Track positions with color codes subtracted - offsets in the final text preview result + let matchTextStartRealIdx = -1; + let textRealIdx = 0; + + const realTextParts: string[] = []; + + for (let i = 0; i < text.length - (RipgrepEngine.MATCH_END_MARKER.length - 1); i++) { + if (text.substr(i, RipgrepEngine.MATCH_START_MARKER.length) === RipgrepEngine.MATCH_START_MARKER) { + // Match start + const chunk = text.slice(lastMatchEndPos, i); + realTextParts.push(chunk); + i += RipgrepEngine.MATCH_START_MARKER.length; + matchTextStartPos = i; + matchTextStartRealIdx = textRealIdx; + } else if (text.substr(i, RipgrepEngine.MATCH_END_MARKER.length) === RipgrepEngine.MATCH_END_MARKER) { + // Match end + const chunk = text.slice(matchTextStartPos, i); + realTextParts.push(chunk); + lineMatch.addMatch(matchTextStartRealIdx, textRealIdx - matchTextStartRealIdx); + matchTextStartPos = -1; + matchTextStartRealIdx = -1; + i += RipgrepEngine.MATCH_END_MARKER.length; + lastMatchEndPos = i; + this.numResults++; + + if (this.numResults >= this.config.maxResults) { + this.cancel(); + onResult([fileMatch.serialize()]); + done(null, { + limitHit: true, + stats: null + }); + } + } else { + // blank line + } + + textRealIdx++; + } + + const chunk = text.slice(lastMatchEndPos); + realTextParts.push(chunk); + + const preview = realTextParts.join(''); + lineMatch.preview = preview; + } else { + r = outputLine.match(RipgrepEngine.FILE_REGEX); + if (r) { + // Line is a file path - send all collected results for the previous file path + if (fileMatch) { + // Check fileMatch against other exclude globs, and fix numResults + onResult([fileMatch.serialize()]); + } + + fileMatch = new FileMatch(path.join(rootFolder, r[1])); + } else { + // Line is empty (or malformed) + } + } + } + }); + + this.rgProc.stderr.on('data', data => { + console.log('stderr'); + console.log(data.toString()); + }); + + this.rgProc.on('close', code => { + this.rgProc = null; + console.log(`closed with ${code}`); + if (fileMatch) { + console.log(`calling onResult`); + onResult([fileMatch.serialize()]); + } + + if (!this.isDone) { + this.isDone = true; + done(null, { + limitHit: false, + stats: null + }); + } + }); + } + + private getRgArgs(): string[] { + const args = ['--heading', '-uu', '--line-number', '--color', 'ansi', '--colors', 'path:none', '--colors', 'line:none', '--colors', 'match:fg:red', '--colors', 'match:style:nobold']; // -uu == Skip gitignore files, and hidden files/folders + args.push(this.config.contentPattern.isCaseSensitive ? '--case-sensitive' : '--ignore-case'); + + // TODO: extraFiles, filePattern ? + if (this.config.includePattern) { + Object.keys(this.config.includePattern).forEach(inclKey => { + const inclValue = this.config.includePattern[inclKey]; + if (typeof inclValue === 'boolean' && inclValue) { + args.push('-g', inclKey); + } else if (inclValue && inclValue.when) { + // Possible? + } + }); + } + + // TODO, -g excludes globs that match anywhere within the path. Our globs should match from the root. + if (this.config.excludePattern) { + Object.keys(this.config.excludePattern).forEach(exclKey => { + const exclValue = this.config.excludePattern[exclKey]; + if (typeof exclValue === 'boolean' && exclValue) { + args.push('-g', `!${exclKey}`); + } else if (exclValue && exclValue.when) { + this.postProcessExclusions.push(exclValue); + } + }); + } + + if (this.config.contentPattern.isRegExp) { + if (this.config.contentPattern.isWordMatch) { + args.push('--word-regexp'); + } + + args.push('--regexp', this.config.contentPattern.pattern); + } else { + if (this.config.contentPattern.isWordMatch) { + args.push('--word-regexp', '--regexp', strings.escapeRegExpCharacters(this.config.contentPattern.pattern)); + } else { + args.push('--fixed-strings', this.config.contentPattern.pattern); + } + } + + return args; + } +} + + +export class FileMatch implements ISerializedFileMatch { + path: string; + lineMatches: LineMatch[]; + + constructor(path: string) { + this.path = path; + this.lineMatches = []; + } + + addMatch(lineMatch: LineMatch): void { + this.lineMatches.push(lineMatch); + } + + isEmpty(): boolean { + return this.lineMatches.length === 0; + } + + serialize(): ISerializedFileMatch { + let lineMatches: ILineMatch[] = []; + let numMatches = 0; + + for (let i = 0; i < this.lineMatches.length; i++) { + numMatches += this.lineMatches[i].offsetAndLengths.length; + lineMatches.push(this.lineMatches[i].serialize()); + } + + return { + path: this.path, + lineMatches, + numMatches + }; + } +} + +export class LineMatch implements ILineMatch { + preview: string; + lineNumber: number; + offsetAndLengths: number[][]; + + constructor(preview: string, lineNumber: number) { + this.preview = preview.replace(/(\r|\n)*$/, ''); + this.lineNumber = lineNumber; + this.offsetAndLengths = []; + } + + getText(): string { + return this.preview; + } + + getLineNumber(): number { + return this.lineNumber; + } + + addMatch(offset: number, length: number): void { + this.offsetAndLengths.push([offset, length]); + } + + serialize(): ILineMatch { + const result = { + preview: this.preview, + lineNumber: this.lineNumber, + offsetAndLengths: this.offsetAndLengths + }; + + return result; + } +} \ No newline at end of file diff --git a/src/vs/workbench/services/search/node/searchService.ts b/src/vs/workbench/services/search/node/searchService.ts index 9ea10c7de733701a6a7d5dd87b8be6ab2dc40d9f..a5b3f8afbeb075787846991a8991efbb20bf42ee 100644 --- a/src/vs/workbench/services/search/node/searchService.ts +++ b/src/vs/workbench/services/search/node/searchService.ts @@ -215,7 +215,8 @@ export class DiskSearch { AMD_ENTRYPOINT: 'vs/workbench/services/search/node/searchApp', PIPE_LOGGING: 'true', VERBOSE_LOGGING: verboseLogging - } + }, + debug: 7890 } );