Ripgrep search

.vscode/launch.json

@@ -63,6 +63,16 @@
 				"${workspaceRoot}/out/**/*.js"
 			]
 		},
+		{
+			"type": "node",
+			"request": "attach",
+			"name": "Attach to Search process",
+			"port": 7890,
+			"sourceMaps": true,
+			"outFiles": [
+				"${workspaceRoot}/out/**/*.js"
+			]
+		},
 		{
 			"type": "node",
 			"request": "attach",

npm-shrinkwrap.json

@@ -410,9 +410,9 @@
       "resolved": "https://registry.npmjs.org/vscode-debugprotocol/-/vscode-debugprotocol-1.17.0.tgz"
     },
     "vscode-ripgrep": {
-      "version": "0.0.8",
-      "from": "vscode-ripgrep@0.0.8",
-      "resolved": "https://registry.npmjs.org/vscode-ripgrep/-/vscode-ripgrep-0.0.8.tgz"
+      "version": "0.0.10",
+      "from": "vscode-ripgrep@0.0.10",
+      "resolved": "https://registry.npmjs.org/vscode-ripgrep/-/vscode-ripgrep-0.0.10.tgz"
     },
     "vscode-textmate": {
       "version": "3.1.1",

package.json

@@ -35,7 +35,7 @@
     "semver": "4.3.6",
     "v8-profiler": "jrieken/v8-profiler#vscode",
     "vscode-debugprotocol": "1.17.0",
-    "vscode-ripgrep": "0.0.8",
+    "vscode-ripgrep": "0.0.10",
     "vscode-textmate": "^3.1.1",
     "winreg": "1.2.0",
     "xterm": "Tyriar/xterm.js#vscode-release/1.11",

src/typings/vscode-ripgrep.d.ts

+declare module 'vscode-ripgrep' {
+	export const rgPath: string;
+}

src/vs/workbench/parts/search/browser/searchViewlet.ts

@@ -35,7 +35,7 @@ import { Viewlet } from 'vs/workbench/browser/viewlet';
 import { Match, FileMatch, SearchModel, FileMatchOrMatch, IChangeEvent, ISearchWorkbenchService } from 'vs/workbench/parts/search/common/searchModel';
 import { QueryBuilder } from 'vs/workbench/parts/search/common/searchQuery';
 import { MessageType, InputBox } from 'vs/base/browser/ui/inputbox/inputBox';
-import { getExcludes, ISearchProgressItem, ISearchComplete, ISearchQuery, IQueryOptions, ISearchConfiguration } from 'vs/platform/search/common/search';
+import { getExcludes, ISearchComplete, ISearchQuery, IQueryOptions, ISearchConfiguration } from 'vs/platform/search/common/search';
 import { IWorkbenchEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { IStorageService, StorageScope } from 'vs/platform/storage/common/storage';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
@@ -988,10 +988,7 @@ export class SearchViewlet extends Viewlet {
 	private onQueryTriggered(query: ISearchQuery, excludePattern: string, includePattern: string): void {
 		this.viewModel.cancelSearch();
 
-		// Progress total is 100.0% for more progress bar granularity
-		let progressTotal = 1000;
-		let progressRunner = this.progressService.show(progressTotal);
-		let progressWorked = 0;
+		let progressRunner = this.progressService.show(/*infinite=*/true);
 
 		this.loading = true;
 		this.searchWidget.searchInput.clearMessage();
@@ -1015,14 +1012,7 @@ export class SearchViewlet extends Viewlet {
 		let isDone = false;
 		let onComplete = (completed?: ISearchComplete) => {
 			isDone = true;
-
-			// Complete up to 100% as needed
-			if (completed) {
-				progressRunner.worked(progressTotal - progressWorked);
-				setTimeout(() => progressRunner.done(), 200);
-			} else {
-				progressRunner.done();
-			}
+			progressRunner.done();
 
 			this.onSearchResultsChanged().then(() => autoExpand(true));
 			this.viewModel.replaceString = this.searchWidget.getReplaceValue();
@@ -1126,18 +1116,7 @@ export class SearchViewlet extends Viewlet {
 			}
 		};
 
-		let total: number = 0;
-		let worked: number = 0;
 		let visibleMatches = 0;
-		let onProgress = (p: ISearchProgressItem) => {
-			// Progress
-			if (p.total) {
-				total = p.total;
-			}
-			if (p.worked) {
-				worked = p.worked;
-			}
-		};
 
 		// Handle UI updates in an interval to show frequent progress and results
 		let uiRefreshHandle = setInterval(() => {
@@ -1146,28 +1125,6 @@ export class SearchViewlet extends Viewlet {
 				return;
 			}
 
-			// Progress bar update
-			let fakeProgress = true;
-			if (total > 0 && worked > 0) {
-				let ratio = Math.round((worked / total) * progressTotal);
-				if (ratio > progressWorked) { // never show less progress than what we have already
-					progressRunner.worked(ratio - progressWorked);
-					progressWorked = ratio;
-					fakeProgress = false;
-				}
-			}
-
-			// Fake progress up to 90%, or when actual progress beats it
-			const fakeMax = 900;
-			const fakeMultiplier = 12;
-			if (fakeProgress && progressWorked < fakeMax) {
-				// Linearly decrease the rate of fake progress.
-				// 1 is the smallest allowed amount of progress.
-				const fakeAmt = Math.round((fakeMax - progressWorked) / fakeMax * fakeMultiplier) || 1;
-				progressWorked += fakeAmt;
-				progressRunner.worked(fakeAmt);
-			}
-
 			// Search result tree update
 			const fileCount = this.viewModel.searchResult.fileCount();
 			if (visibleMatches !== fileCount) {
@@ -1188,7 +1145,7 @@ export class SearchViewlet extends Viewlet {
 
 		this.searchWidget.setReplaceAllActionState(false);
 		// this.replaceService.disposeAllReplacePreviews();
-		this.viewModel.search(query).done(onComplete, onError, onProgress);
+		this.viewModel.search(query).done(onComplete, onError);
 	}
 
 	private updateSearchResultCount(): void {

src/vs/workbench/services/search/node/rawSearchService.ts

@@ -17,10 +17,8 @@ import objects = require('vs/base/common/objects');
 import scorer = require('vs/base/common/scorer');
 import strings = require('vs/base/common/strings');
 import { PPromise, TPromise } from 'vs/base/common/winjs.base';
-import { MAX_FILE_SIZE } from 'vs/platform/files/common/files';
-import { FileWalker, Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch';
-import { Engine as TextSearchEngine } from 'vs/workbench/services/search/node/textSearch';
-import { TextSearchWorkerProvider } from 'vs/workbench/services/search/node/textSearchWorkerProvider';
+import { Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch';
+import { RipgrepEngine } from 'vs/workbench/services/search/node/ripgrepTextSearch';
 import { IRawSearchService, IRawSearch, IRawFileMatch, ISerializedFileMatch, ISerializedSearchProgressItem, ISerializedSearchComplete, ISearchEngine } from './search';
 import { ICachedSearchStats, IProgress } from 'vs/platform/search/common/search';
 
@@ -32,29 +30,12 @@ export class SearchService implements IRawSearchService {
 
 	private caches: { [cacheKey: string]: Cache; } = Object.create(null);
 
-	private textSearchWorkerProvider: TextSearchWorkerProvider;
-
 	public fileSearch(config: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem> {
 		return this.doFileSearch(FileSearchEngine, config, SearchService.BATCH_SIZE);
 	}
 
 	public textSearch(config: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem> {
-		if (!this.textSearchWorkerProvider) {
-			this.textSearchWorkerProvider = new TextSearchWorkerProvider();
-		}
-
-		let engine = new TextSearchEngine(
-			config,
-			new FileWalker({
-				rootFolders: config.rootFolders,
-				extraFiles: config.extraFiles,
-				includePattern: config.includePattern,
-				excludePattern: config.excludePattern,
-				filePattern: config.filePattern,
-				maxFilesize: MAX_FILE_SIZE
-			}),
-			this.textSearchWorkerProvider);
-
+		let engine = new RipgrepEngine(config);
 		return this.doTextSearch(engine, SearchService.BATCH_SIZE);
 	}
 
@@ -279,7 +260,7 @@ export class SearchService implements IRawSearchService {
 		});
 	}
 
-	private doTextSearch(engine: TextSearchEngine, batchSize: number): PPromise<ISerializedSearchComplete, IRawProgressItem<ISerializedFileMatch>> {
+	private doTextSearch(engine: RipgrepEngine, batchSize: number): PPromise<ISerializedSearchComplete, IRawProgressItem<ISerializedFileMatch>> {
 		return new PPromise<ISerializedSearchComplete, IRawProgressItem<ISerializedFileMatch>>((c, e, p) => {
 			// Use BatchedCollector to get new results to the frontend every 2s at least, until 50 results have been returned
 			const collector = new BatchedCollector<ISerializedFileMatch>(batchSize, p);

src/vs/workbench/services/search/node/ripgrepTextSearch.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as cp from 'child_process';
+import * as path from 'path';
+import { rgPath } from 'vscode-ripgrep';
+
+import * as strings from 'vs/base/common/strings';
+import * as glob from 'vs/base/common/glob';
+import { ILineMatch, IProgress } from 'vs/platform/search/common/search';
+
+import { ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine } from './search';
+
+export class RipgrepEngine implements ISearchEngine<ISerializedFileMatch[]> {
+	private static RESULT_REGEX = /^\u001b\[m(\d+)\u001b\[m:(.*)$/;
+	private static FILE_REGEX = /^\u001b\[m(.+)\u001b\[m$/;
+
+	private static MATCH_START_MARKER = '\u001b[m\u001b[31m';
+	private static MATCH_END_MARKER = '\u001b[m';
+
+	private config: IRawSearch;
+	private isDone = false;
+	private rgProc: cp.ChildProcess;
+	private postProcessExclusions: glob.SiblingClause[] = [];
+
+	private numResults = 0;
+
+	constructor(config: IRawSearch) {
+		this.config = config;
+	}
+
+	cancel(): void {
+		this.isDone = true;
+
+		this.rgProc.kill();
+	}
+
+	search(onResult: (match: ISerializedFileMatch[]) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
+		this.searchNextFolder(onResult, onProgress, done);
+	}
+
+	private searchNextFolder(onResult: (match: ISerializedFileMatch[]) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
+		if (this.config.rootFolders.length) {
+			// TODO search all root folders
+			this.searchFolder(this.config.rootFolders.shift(), onResult, onProgress, done);
+		}
+	}
+
+	private searchFolder(rootFolder: string, onResult: (match: ISerializedFileMatch[]) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
+		const rgArgs = this.getRgArgs();
+		console.log(`rg ${rgArgs.join(' ')}, cwd: ${rootFolder}`);
+		this.rgProc = cp.spawn(rgPath, rgArgs, { cwd: rootFolder });
+
+		let fileMatch: FileMatch;
+		let remainder: string;
+		this.rgProc.stdout.on('data', data => {
+			// If the previous data chunk didn't end in a newline, append it to this chunk
+			const dataStr = remainder ?
+				remainder + data.toString() :
+				data.toString();
+
+			const dataLines: string[] = dataStr.split('\n');
+			remainder = dataLines.pop();
+
+			for (let l = 0; l < dataLines.length; l++) {
+				const outputLine = dataLines[l];
+				if (this.isDone) {
+					break;
+				}
+
+				let r = outputLine.match(RipgrepEngine.RESULT_REGEX);
+				if (r) {
+					// Line is a result - add to collected results for the current file path
+					const line = parseInt(r[1]) - 1;
+					const text = r[2];
+
+					const lineMatch = new LineMatch(text, line);
+					fileMatch.addMatch(lineMatch);
+
+					let lastMatchEndPos = 0;
+					let matchTextStartPos = -1;
+
+					// Track positions with color codes subtracted - offsets in the final text preview result
+					let matchTextStartRealIdx = -1;
+					let textRealIdx = 0;
+
+					const realTextParts: string[] = [];
+
+					for (let i = 0; i < text.length - (RipgrepEngine.MATCH_END_MARKER.length - 1); i++) {
+						if (text.substr(i, RipgrepEngine.MATCH_START_MARKER.length) === RipgrepEngine.MATCH_START_MARKER) {
+							// Match start
+							const chunk = text.slice(lastMatchEndPos, i);
+							realTextParts.push(chunk);
+							i += RipgrepEngine.MATCH_START_MARKER.length;
+							matchTextStartPos = i;
+							matchTextStartRealIdx = textRealIdx;
+						} else if (text.substr(i, RipgrepEngine.MATCH_END_MARKER.length) === RipgrepEngine.MATCH_END_MARKER) {
+							// Match end
+							const chunk = text.slice(matchTextStartPos, i);
+							realTextParts.push(chunk);
+							lineMatch.addMatch(matchTextStartRealIdx, textRealIdx - matchTextStartRealIdx);
+							matchTextStartPos = -1;
+							matchTextStartRealIdx = -1;
+							i += RipgrepEngine.MATCH_END_MARKER.length;
+							lastMatchEndPos = i;
+							this.numResults++;
+
+							if (this.numResults >= this.config.maxResults) {
+								this.cancel();
+								onResult([fileMatch.serialize()]);
+								done(null, {
+									limitHit: true,
+									stats: null
+								});
+							}
+						} else {
+							// blank line
+						}
+
+						textRealIdx++;
+					}
+
+					const chunk = text.slice(lastMatchEndPos);
+					realTextParts.push(chunk);
+
+					const preview = realTextParts.join('');
+					lineMatch.preview = preview;
+				} else {
+					r = outputLine.match(RipgrepEngine.FILE_REGEX);
+					if (r) {
+						// Line is a file path - send all collected results for the previous file path
+						if (fileMatch) {
+							// Check fileMatch against other exclude globs, and fix numResults
+							onResult([fileMatch.serialize()]);
+						}
+
+						fileMatch = new FileMatch(path.join(rootFolder, r[1]));
+					} else {
+						// Line is empty (or malformed)
+					}
+				}
+			}
+		});
+
+		this.rgProc.stderr.on('data', data => {
+			console.log('stderr');
+			console.log(data.toString());
+		});
+
+		this.rgProc.on('close', code => {
+			this.rgProc = null;
+			console.log(`closed with ${code}`);
+			if (fileMatch) {
+				console.log(`calling onResult`);
+				onResult([fileMatch.serialize()]);
+			}
+
+			if (!this.isDone) {
+				this.isDone = true;
+				done(null, {
+					limitHit: false,
+					stats: null
+				});
+			}
+		});
+	}
+
+	private getRgArgs(): string[] {
+		const args = ['--heading', '-uu', '--line-number', '--color', 'ansi', '--colors', 'path:none', '--colors', 'line:none', '--colors', 'match:fg:red', '--colors', 'match:style:nobold']; // -uu == Skip gitignore files, and hidden files/folders
+		args.push(this.config.contentPattern.isCaseSensitive ? '--case-sensitive' : '--ignore-case');
+
+		// TODO: extraFiles, filePattern ?
+		if (this.config.includePattern) {
+			Object.keys(this.config.includePattern).forEach(inclKey => {
+				const inclValue = this.config.includePattern[inclKey];
+				if (typeof inclValue === 'boolean' && inclValue) {
+					args.push('-g', inclKey);
+				} else if (inclValue && inclValue.when) {
+					// Possible?
+				}
+			});
+		}
+
+		// TODO, -g excludes globs that match anywhere within the path. Our globs should match from the root.
+		if (this.config.excludePattern) {
+			Object.keys(this.config.excludePattern).forEach(exclKey => {
+				const exclValue = this.config.excludePattern[exclKey];
+				if (typeof exclValue === 'boolean' && exclValue) {
+					args.push('-g', `!${exclKey}`);
+				} else if (exclValue && exclValue.when) {
+					this.postProcessExclusions.push(exclValue);
+				}
+			});
+		}
+
+		if (this.config.contentPattern.isRegExp) {
+			if (this.config.contentPattern.isWordMatch) {
+				args.push('--word-regexp');
+			}
+
+			args.push('--regexp', this.config.contentPattern.pattern);
+		} else {
+			if (this.config.contentPattern.isWordMatch) {
+				args.push('--word-regexp', '--regexp', strings.escapeRegExpCharacters(this.config.contentPattern.pattern));
+			} else {
+				args.push('--fixed-strings', this.config.contentPattern.pattern);
+			}
+		}
+
+		return args;
+	}
+}
+
+
+export class FileMatch implements ISerializedFileMatch {
+	path: string;
+	lineMatches: LineMatch[];
+
+	constructor(path: string) {
+		this.path = path;
+		this.lineMatches = [];
+	}
+
+	addMatch(lineMatch: LineMatch): void {
+		this.lineMatches.push(lineMatch);
+	}
+
+	isEmpty(): boolean {
+		return this.lineMatches.length === 0;
+	}
+
+	serialize(): ISerializedFileMatch {
+		let lineMatches: ILineMatch[] = [];
+		let numMatches = 0;
+
+		for (let i = 0; i < this.lineMatches.length; i++) {
+			numMatches += this.lineMatches[i].offsetAndLengths.length;
+			lineMatches.push(this.lineMatches[i].serialize());
+		}
+
+		return {
+			path: this.path,
+			lineMatches,
+			numMatches
+		};
+	}
+}
+
+export class LineMatch implements ILineMatch {
+	preview: string;
+	lineNumber: number;
+	offsetAndLengths: number[][];
+
+	constructor(preview: string, lineNumber: number) {
+		this.preview = preview.replace(/(\r|\n)*$/, '');
+		this.lineNumber = lineNumber;
+		this.offsetAndLengths = [];
+	}
+
+	getText(): string {
+		return this.preview;
+	}
+
+	getLineNumber(): number {
+		return this.lineNumber;
+	}
+
+	addMatch(offset: number, length: number): void {
+		this.offsetAndLengths.push([offset, length]);
+	}
+
+	serialize(): ILineMatch {
+		const result = {
+			preview: this.preview,
+			lineNumber: this.lineNumber,
+			offsetAndLengths: this.offsetAndLengths
+		};
+
+		return result;
+	}
+}
\ No newline at end of file

src/vs/workbench/services/search/node/searchService.ts

@@ -215,7 +215,8 @@ export class DiskSearch {
 					AMD_ENTRYPOINT: 'vs/workbench/services/search/node/searchApp',
 					PIPE_LOGGING: 'true',
 					VERBOSE_LOGGING: verboseLogging
-				}
+				},
+				debug: 7890
 			}
 		);