fileSearch.ts 24.2 KB
Newer Older
E
Erich Gamma 已提交
1 2 3 4 5
/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

C
Christof Marti 已提交
6
import * as childProcess from 'child_process';
7
import * as fs from 'fs';
8
import * as path from 'vs/base/common/path';
B
Benjamin Pasero 已提交
9
import { Readable } from 'stream';
10
import { StringDecoder } from 'string_decoder';
11
import * as arrays from 'vs/base/common/arrays';
12 13 14 15
import { toErrorMessage } from 'vs/base/common/errorMessage';
import * as glob from 'vs/base/common/glob';
import * as normalization from 'vs/base/common/normalization';
import * as objects from 'vs/base/common/objects';
B
Benjamin Pasero 已提交
16
import { isEqualOrParent } from 'vs/base/common/extpath';
17
import * as platform from 'vs/base/common/platform';
18
import { StopWatch } from 'vs/base/common/stopwatch';
19 20
import * as strings from 'vs/base/common/strings';
import * as types from 'vs/base/common/types';
21
import { URI } from 'vs/base/common/uri';
B
Benjamin Pasero 已提交
22
import { readdir } from 'vs/base/node/pfs';
23
import { IFileQuery, IFolderQuery, IProgressMessage, ISearchEngineStats, IRawFileMatch, ISearchEngine, ISearchEngineSuccess, isFilePatternMatch } from 'vs/workbench/services/search/common/search';
24
import { spawnRipgrepCmd } from './ripgrepFileSearch';
R
Rob Lourens 已提交
25
import { prepareQuery } from 'vs/base/parts/quickopen/common/quickOpenScorer';
E
Erich Gamma 已提交
26

C
Christof Marti 已提交
27
interface IDirectoryEntry {
28
	base: string;
C
Christof Marti 已提交
29 30 31 32 33 34 35 36 37
	relativePath: string;
	basename: string;
}

interface IDirectoryTree {
	rootEntries: IDirectoryEntry[];
	pathToEntries: { [relativePath: string]: IDirectoryEntry[] };
}

38 39 40 41 42
const killCmds = new Set<() => void>();
process.on('exit', () => {
	killCmds.forEach(cmd => cmd());
});

E
Erich Gamma 已提交
43
export class FileWalker {
44
	private config: IFileQuery;
45
	private filePattern: string;
46
	private normalizedFilePatternLowercase: string | null = null;
47 48
	private includePattern: glob.ParsedExpression | undefined;
	private maxResults: number | null;
49
	private exists: boolean;
50
	private maxFilesize: number | null = null;
E
Erich Gamma 已提交
51 52
	private isLimitHit: boolean;
	private resultCount: number;
53 54
	private isCanceled = false;
	private fileWalkSW: StopWatch | null = null;
C
chrmarti 已提交
55 56
	private directoriesWalked: number;
	private filesWalked: number;
C
Christof Marti 已提交
57
	private errors: string[];
58 59
	private cmdSW: StopWatch | null = null;
	private cmdResultCount: number = 0;
E
Erich Gamma 已提交
60

61
	private folderExcludePatterns: Map<string, AbsoluteAndRelativeParsedExpression>;
62
	private globalExcludePattern: glob.ParsedExpression | undefined;
63

E
Erich Gamma 已提交
64 65
	private walkedPaths: { [path: string]: boolean; };

R
Rob Lourens 已提交
66
	constructor(config: IFileQuery) {
E
Erich Gamma 已提交
67
		this.config = config;
68
		this.filePattern = config.filePattern || '';
C
Christof Marti 已提交
69
		this.includePattern = config.includePattern && glob.parse(config.includePattern);
E
Erich Gamma 已提交
70
		this.maxResults = config.maxResults || null;
71
		this.exists = !!config.exists;
E
Erich Gamma 已提交
72
		this.walkedPaths = Object.create(null);
73 74
		this.resultCount = 0;
		this.isLimitHit = false;
C
chrmarti 已提交
75 76
		this.directoriesWalked = 0;
		this.filesWalked = 0;
C
Christof Marti 已提交
77
		this.errors = [];
B
Benjamin Pasero 已提交
78

79
		if (this.filePattern) {
80
			this.normalizedFilePatternLowercase = prepareQuery(this.filePattern).lowercase;
B
Benjamin Pasero 已提交
81
		}
82 83

		this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);
84
		this.folderExcludePatterns = new Map<string, AbsoluteAndRelativeParsedExpression>();
85 86

		config.folderQueries.forEach(folderQuery => {
87
			const folderExcludeExpression: glob.IExpression = objects.assign({}, folderQuery.excludePattern || {}, this.config.excludePattern || {});
88 89

			// Add excludes for other root folders
90
			const fqPath = folderQuery.folder.fsPath;
91
			config.folderQueries
92 93
				.map(rootFolderQuery => rootFolderQuery.folder.fsPath)
				.filter(rootFolder => rootFolder !== fqPath)
94 95
				.forEach(otherRootFolder => {
					// Exclude nested root folders
96 97
					if (isEqualOrParent(otherRootFolder, fqPath)) {
						folderExcludeExpression[path.relative(fqPath, otherRootFolder)] = true;
98
					}
99 100
				});

101
			this.folderExcludePatterns.set(fqPath, new AbsoluteAndRelativeParsedExpression(folderExcludeExpression, fqPath));
102
		});
E
Erich Gamma 已提交
103 104
	}

R
Rob Lourens 已提交
105
	cancel(): void {
E
Erich Gamma 已提交
106 107 108
		this.isCanceled = true;
	}

109
	walk(folderQueries: IFolderQuery[], extraFiles: URI[], onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgressMessage) => void, done: (error: Error | null, isLimitHit: boolean) => void): void {
110
		this.fileWalkSW = StopWatch.create(false);
E
Erich Gamma 已提交
111

112
		// Support that the file pattern is a full path to a file that exists
113 114 115
		if (this.isCanceled) {
			return done(null, this.isLimitHit);
		}
E
Erich Gamma 已提交
116

117
		// For each extra file
118 119 120 121 122
		extraFiles.forEach(extraFilePath => {
			const basename = path.basename(extraFilePath.fsPath);
			if (this.globalExcludePattern && this.globalExcludePattern(extraFilePath.fsPath, basename)) {
				return; // excluded
			}
E
Erich Gamma 已提交
123

124 125 126
			// File: Check for match on file pattern and include pattern
			this.matchFile(onResult, { relativePath: extraFilePath.fsPath /* no workspace relative path */, basename });
		});
127

R
Rob Lourens 已提交
128
		this.cmdSW = StopWatch.create(false);
C
Christof Marti 已提交
129

130
		// For each root folder
B
Benjamin Pasero 已提交
131
		this.parallel<IFolderQuery, void>(folderQueries, (folderQuery: IFolderQuery, rootFolderDone: (err: Error | null, result: void) => void) => {
R
Rob Lourens 已提交
132
			this.call(this.cmdTraversal, this, folderQuery, onResult, onMessage, (err?: Error) => {
133 134 135 136 137 138
				if (err) {
					const errorMessage = toErrorMessage(err);
					console.error(errorMessage);
					this.errors.push(errorMessage);
					rootFolderDone(err, undefined);
				} else {
139
					rootFolderDone(null, undefined);
140
				}
C
Christof Marti 已提交
141
			});
142 143
		}, (errors, _result) => {
			this.fileWalkSW!.stop();
M
Matt Bierner 已提交
144
			const err = errors ? arrays.coalesce(errors)[0] : null;
145
			done(err, this.isLimitHit);
C
Christof Marti 已提交
146 147
		});
	}
148

B
Benjamin Pasero 已提交
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
	private parallel<T, E>(list: T[], fn: (item: T, callback: (err: Error | null, result: E | null) => void) => void, callback: (err: Array<Error | null> | null, result: E[]) => void): void {
		const results = new Array(list.length);
		const errors = new Array<Error | null>(list.length);
		let didErrorOccur = false;
		let doneCount = 0;

		if (list.length === 0) {
			return callback(null, []);
		}

		list.forEach((item, index) => {
			fn(item, (error, result) => {
				if (error) {
					didErrorOccur = true;
					results[index] = null;
					errors[index] = error;
				} else {
					results[index] = result;
					errors[index] = null;
				}

				if (++doneCount === list.length) {
					return callback(didErrorOccur ? errors : null, results);
				}
			});
		});
	}

177
	private call<F extends Function>(fun: F, that: any, ...args: any[]): void {
C
Christof Marti 已提交
178 179 180 181 182 183 184
		try {
			fun.apply(that, args);
		} catch (e) {
			args[args.length - 1](e);
		}
	}

185
	private cmdTraversal(folderQuery: IFolderQuery, onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgressMessage) => void, cb: (err?: Error) => void): void {
186
		const rootFolder = folderQuery.folder.fsPath;
187
		const isMac = platform.isMacintosh;
188 189
		let cmd: childProcess.ChildProcess;
		const killCmd = () => cmd && cmd.kill();
190
		killCmds.add(killCmd);
191

192
		let done = (err?: Error) => {
193
			killCmds.delete(killCmd);
J
Johannes Rieken 已提交
194
			done = () => { };
195 196 197 198
			cb(err);
		};
		let leftover = '';
		const tree = this.initDirectoryTree();
199 200

		let noSiblingsClauses: boolean;
R
Rob Lourens 已提交
201 202 203 204 205 206 207 208 209 210 211
		const ripgrep = spawnRipgrepCmd(this.config, folderQuery, this.config.includePattern, this.folderExcludePatterns.get(folderQuery.folder.fsPath)!.expression);
		cmd = ripgrep.cmd;
		noSiblingsClauses = !Object.keys(ripgrep.siblingClauses).length;

		const escapedArgs = ripgrep.rgArgs.args
			.map(arg => arg.match(/^-/) ? arg : `'${arg}'`)
			.join(' ');

		let rgCmd = `rg ${escapedArgs}\n - cwd: ${ripgrep.cwd}`;
		if (ripgrep.rgArgs.siblingClauses) {
			rgCmd += `\n - Sibling clauses: ${JSON.stringify(ripgrep.rgArgs.siblingClauses)}`;
212
		}
R
Rob Lourens 已提交
213
		onMessage({ message: rgCmd });
214

215
		this.cmdResultCount = 0;
M
Matt Bierner 已提交
216
		this.collectStdout(cmd, 'utf8', onMessage, (err: Error | null, stdout?: string, last?: boolean) => {
C
Christof Marti 已提交
217 218 219 220
			if (err) {
				done(err);
				return;
			}
221
			if (this.isLimitHit) {
222
				done();
223 224
				return;
			}
E
Erich Gamma 已提交
225

C
Christof Marti 已提交
226
			// Mac: uses NFD unicode form on disk, but we want NFC
227
			const normalized = leftover + (isMac ? normalization.normalizeNFC(stdout || '') : stdout);
R
Rob Lourens 已提交
228
			const relativeFiles = normalized.split('\n');
C
Christof Marti 已提交
229

230 231 232 233 234 235 236
			if (last) {
				const n = relativeFiles.length;
				relativeFiles[n - 1] = relativeFiles[n - 1].trim();
				if (!relativeFiles[n - 1]) {
					relativeFiles.pop();
				}
			} else {
237
				leftover = relativeFiles.pop() || '';
C
Christof Marti 已提交
238 239
			}

240 241 242 243 244
			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
				done(new Error('Splitting up files failed'));
				return;
			}

245 246
			this.cmdResultCount += relativeFiles.length;

R
Rob Lourens 已提交
247
			if (noSiblingsClauses) {
248 249
				for (const relativePath of relativeFiles) {
					const basename = path.basename(relativePath);
250
					this.matchFile(onResult, { base: rootFolder, relativePath, searchPath: this.getSearchPath(folderQuery, relativePath), basename });
251 252 253 254
					if (this.isLimitHit) {
						killCmd();
						break;
					}
255
				}
256
				if (last || this.isLimitHit) {
257
					done();
258
				}
259

260 261 262
				return;
			}

263
			// TODO: Optimize siblings clauses with ripgrep here.
264
			this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
C
Christof Marti 已提交
265

266 267 268
			if (last) {
				this.matchDirectoryTree(tree, rootFolder, onResult);
				done();
269
			}
C
Christof Marti 已提交
270 271 272
		});
	}

273 274 275
	/**
	 * Public for testing.
	 */
R
Rob Lourens 已提交
276
	spawnFindCmd(folderQuery: IFolderQuery) {
277
		const excludePattern = this.folderExcludePatterns.get(folderQuery.folder.fsPath)!;
278 279
		const basenames = excludePattern.getBasenameTerms();
		const pathTerms = excludePattern.getPathTerms();
280
		const args = ['-L', '.'];
281
		if (basenames.length || pathTerms.length) {
282
			args.push('-not', '(', '(');
283
			for (const basename of basenames) {
284
				args.push('-name', basename);
285 286
				args.push('-o');
			}
287
			for (const path of pathTerms) {
288
				args.push('-path', path);
289
				args.push('-o');
290
			}
291
			args.pop();
292 293 294
			args.push(')', '-prune', ')');
		}
		args.push('-type', 'f');
295
		return childProcess.spawn('find', args, { cwd: folderQuery.folder.fsPath });
296 297 298 299 300
	}

	/**
	 * Public for testing.
	 */
R
Rob Lourens 已提交
301
	readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error | null, stdout?: string) => void): void {
302
		let all = '';
M
Matt Bierner 已提交
303
		this.collectStdout(cmd, encoding, () => { }, (err: Error | null, stdout?: string, last?: boolean) => {
304 305 306 307 308 309 310 311 312 313 314 315
			if (err) {
				cb(err);
				return;
			}

			all += stdout;
			if (last) {
				cb(null, all);
			}
		});
	}

316
	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, onMessage: (message: IProgressMessage) => void, cb: (err: Error | null, stdout?: string, last?: boolean) => void): void {
317
		let onData = (err: Error | null, stdout?: string, last?: boolean) => {
318
			if (err || last) {
319
				onData = () => { };
320

321 322 323
				if (this.cmdSW) {
					this.cmdSW.stop();
				}
324 325
			}
			cb(err, stdout, last);
C
Christof Marti 已提交
326 327
		};

328
		let gotData = false;
329 330 331
		if (cmd.stdout) {
			// Should be non-null, but #38195
			this.forwardData(cmd.stdout, encoding, onData);
332
			cmd.stdout.once('data', () => gotData = true);
333 334 335 336
		} else {
			onMessage({ message: 'stdout is null' });
		}

337 338 339 340 341 342 343
		let stderr: Buffer[];
		if (cmd.stderr) {
			// Should be non-null, but #38195
			stderr = this.collectData(cmd.stderr);
		} else {
			onMessage({ message: 'stderr is null' });
		}
344

345
		cmd.on('error', (err: Error) => {
346
			onData(err);
C
Christof Marti 已提交
347 348
		});

349
		cmd.on('close', (code: number) => {
350
			// ripgrep returns code=1 when no results are found
351
			let stderrText: string;
R
Rob Lourens 已提交
352
			if (!gotData && (stderrText = this.decodeData(stderr, encoding)) && rgErrorMsgForDisplay(stderrText)) {
353
				onData(new Error(`command failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
C
Christof Marti 已提交
354
			} else {
R
Rob Lourens 已提交
355
				if (this.exists && code === 0) {
356 357
					this.isLimitHit = true;
				}
358
				onData(null, '', true);
C
Christof Marti 已提交
359 360 361 362
			}
		});
	}

363
	private forwardData(stream: Readable, encoding: string, cb: (err: Error | null, stdout?: string) => void): StringDecoder {
364 365 366 367 368 369 370
		const decoder = new StringDecoder(encoding);
		stream.on('data', (data: Buffer) => {
			cb(null, decoder.write(data));
		});
		return decoder;
	}

C
Christof Marti 已提交
371
	private collectData(stream: Readable): Buffer[] {
372 373
		const buffers: Buffer[] = [];
		stream.on('data', (data: Buffer) => {
C
Christof Marti 已提交
374 375 376 377 378
			buffers.push(data);
		});
		return buffers;
	}

C
Christof Marti 已提交
379 380 381 382 383
	private decodeData(buffers: Buffer[], encoding: string): string {
		const decoder = new StringDecoder(encoding);
		return buffers.map(buffer => decoder.write(buffer)).join('');
	}

384 385 386 387 388 389 390 391 392
	private initDirectoryTree(): IDirectoryTree {
		const tree: IDirectoryTree = {
			rootEntries: [],
			pathToEntries: Object.create(null)
		};
		tree.pathToEntries['.'] = tree.rootEntries;
		return tree;
	}

393
	private addDirectoryEntries({ pathToEntries }: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
394 395
		// Support relative paths to files from a root resource (ignores excludes)
		if (relativeFiles.indexOf(this.filePattern) !== -1) {
396
			const basename = path.basename(this.filePattern);
397
			this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
C
Christof Marti 已提交
398 399
		}

400
		function add(relativePath: string) {
401 402
			const basename = path.basename(relativePath);
			const dirname = path.dirname(relativePath);
C
Christof Marti 已提交
403 404 405 406 407 408
			let entries = pathToEntries[dirname];
			if (!entries) {
				entries = pathToEntries[dirname] = [];
				add(dirname);
			}
			entries.push({
409
				base,
C
Christof Marti 已提交
410 411 412
				relativePath,
				basename
			});
413 414
		}
		relativeFiles.forEach(add);
C
Christof Marti 已提交
415 416
	}

417
	private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
418
		const self = this;
419
		const excludePattern = this.folderExcludePatterns.get(rootFolder)!;
C
Christof Marti 已提交
420 421 422
		const filePattern = this.filePattern;
		function matchDirectory(entries: IDirectoryEntry[]) {
			self.directoriesWalked++;
423
			const hasSibling = glob.hasSiblingFn(() => entries.map(entry => entry.basename));
C
Christof Marti 已提交
424
			for (let i = 0, n = entries.length; i < n; i++) {
425
				const entry = entries[i];
426
				const { relativePath, basename } = entry;
C
Christof Marti 已提交
427 428 429 430 431

				// Check exclude pattern
				// If the user searches for the exact file name, we adjust the glob matching
				// to ignore filtering by siblings because the user seems to know what she
				// is searching for and we want to include the result in that case anyway
432
				if (excludePattern.test(relativePath, basename, filePattern !== basename ? hasSibling : undefined)) {
C
Christof Marti 已提交
433 434
					continue;
				}
E
Erich Gamma 已提交
435

C
Christof Marti 已提交
436 437 438 439 440 441 442 443 444
				const sub = pathToEntries[relativePath];
				if (sub) {
					matchDirectory(sub);
				} else {
					self.filesWalked++;
					if (relativePath === filePattern) {
						continue; // ignore file if its path matches with the file pattern because that is already matched above
					}

445
					self.matchFile(onResult, entry);
C
Christof Marti 已提交
446
				}
447 448 449 450

				if (self.isLimitHit) {
					break;
				}
451
			}
C
Christof Marti 已提交
452 453 454 455
		}
		matchDirectory(rootEntries);
	}

R
Rob Lourens 已提交
456
	getStats(): ISearchEngineStats {
C
chrmarti 已提交
457
		return {
458 459
			cmdTime: this.cmdSW!.elapsed(),
			fileWalkTime: this.fileWalkSW!.elapsed(),
C
chrmarti 已提交
460
			directoriesWalked: this.directoriesWalked,
461
			filesWalked: this.filesWalked,
C
Christof Marti 已提交
462
			cmdResultCount: this.cmdResultCount
C
chrmarti 已提交
463 464 465
		};
	}

466
	private doWalk(folderQuery: IFolderQuery, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error?: Error) => void): void {
467
		const rootFolder = folderQuery.folder;
E
Erich Gamma 已提交
468 469

		// Execute tasks on each file in parallel to optimize throughput
470
		const hasSibling = glob.hasSiblingFn(() => files);
B
Benjamin Pasero 已提交
471
		this.parallel(files, (file: string, clb: (error: Error | null, _?: any) => void): void => {
E
Erich Gamma 已提交
472 473 474

			// Check canceled
			if (this.isCanceled || this.isLimitHit) {
475
				return clb(null);
E
Erich Gamma 已提交
476 477
			}

478
			// Check exclude pattern
E
Erich Gamma 已提交
479 480 481
			// If the user searches for the exact file name, we adjust the glob matching
			// to ignore filtering by siblings because the user seems to know what she
			// is searching for and we want to include the result in that case anyway
482
			const currentRelativePath = relativeParentPath ? [relativeParentPath, file].join(path.sep) : file;
483
			if (this.folderExcludePatterns.get(folderQuery.folder.fsPath)!.test(currentRelativePath, file, this.config.filePattern !== file ? hasSibling : undefined)) {
484
				return clb(null);
E
Erich Gamma 已提交
485 486
			}

487
			// Use lstat to detect links
488
			const currentAbsolutePath = [rootFolder.fsPath, currentRelativePath].join(path.sep);
489
			fs.lstat(currentAbsolutePath, (error, lstat) => {
490
				if (error || this.isCanceled || this.isLimitHit) {
491
					return clb(null);
492
				}
E
Erich Gamma 已提交
493

494
				// If the path is a link, we must instead use fs.stat() to find out if the
495 496 497 498
				// link is a directory or not because lstat will always return the stat of
				// the link which is always a file.
				this.statLinkIfNeeded(currentAbsolutePath, lstat, (error, stat) => {
					if (error || this.isCanceled || this.isLimitHit) {
499
						return clb(null);
500
					}
E
Erich Gamma 已提交
501

502 503
					// Directory: Follow directories
					if (stat.isDirectory()) {
C
chrmarti 已提交
504
						this.directoriesWalked++;
505

506 507
						// to really prevent loops with links we need to resolve the real path of them
						return this.realPathIfNeeded(currentAbsolutePath, lstat, (error, realpath) => {
508
							if (error || this.isCanceled || this.isLimitHit) {
509
								return clb(null);
510 511
							}

512
							realpath = realpath || '';
513
							if (this.walkedPaths[realpath]) {
514
								return clb(null); // escape when there are cycles (can happen with symlinks)
515
							}
E
Erich Gamma 已提交
516

517 518 519
							this.walkedPaths[realpath] = true; // remember as walked

							// Continue walking
B
Benjamin Pasero 已提交
520 521
							return readdir(currentAbsolutePath).then(children => {
								if (this.isCanceled || this.isLimitHit) {
522
									return clb(null);
523 524
								}

525
								this.doWalk(folderQuery, currentRelativePath, children, onResult, err => clb(err || null));
B
Benjamin Pasero 已提交
526 527
							}, error => {
								clb(null);
528 529
							});
						});
530
					}
E
Erich Gamma 已提交
531

532 533
					// File: Check for match on file pattern and include pattern
					else {
C
chrmarti 已提交
534
						this.filesWalked++;
C
Christof Marti 已提交
535
						if (currentRelativePath === this.filePattern) {
536
							return clb(null, undefined); // ignore file if its path matches with the file pattern because checkFilePatternRelativeMatch() takes care of those
537
						}
E
Erich Gamma 已提交
538

539
						if (this.maxFilesize && types.isNumber(stat.size) && stat.size > this.maxFilesize) {
540
							return clb(null, undefined); // ignore file if max file size is hit
541 542
						}

543 544 545 546 547 548 549
						this.matchFile(onResult, {
							base: rootFolder.fsPath,
							relativePath: currentRelativePath,
							searchPath: this.getSearchPath(folderQuery, currentRelativePath),
							basename: file,
							size: stat.size,
						});
550 551 552
					}

					// Unwind
553
					return clb(null, undefined);
554
				});
E
Erich Gamma 已提交
555
			});
M
Matt Bierner 已提交
556 557 558
		}, (error: Array<Error | null> | null): void => {
			const filteredErrors = error ? arrays.coalesce(error) : error; // find any error by removing null values first
			return done(filteredErrors && filteredErrors.length > 0 ? filteredErrors[0] : undefined);
E
Erich Gamma 已提交
559 560 561
		});
	}

562
	private matchFile(onResult: (result: IRawFileMatch) => void, candidate: IRawFileMatch): void {
563
		if (this.isFileMatch(candidate) && (!this.includePattern || this.includePattern(candidate.relativePath, candidate.basename))) {
564 565
			this.resultCount++;

566
			if (this.exists || (this.maxResults && this.resultCount > this.maxResults)) {
567 568 569 570
				this.isLimitHit = true;
			}

			if (!this.isLimitHit) {
571
				onResult(candidate);
572 573 574 575
			}
		}
	}

576
	private isFileMatch(candidate: IRawFileMatch): boolean {
577 578
		// Check for search pattern
		if (this.filePattern) {
579 580 581 582
			if (this.filePattern === '*') {
				return true; // support the all-matching wildcard
			}

583
			if (this.normalizedFilePatternLowercase) {
584
				return isFilePatternMatch(candidate, this.normalizedFilePatternLowercase);
585
			}
586 587 588 589 590 591
		}

		// No patterns means we match all
		return true;
	}

592
	private statLinkIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error | null, stat: fs.Stats) => void): void {
593
		if (lstat.isSymbolicLink()) {
594
			return fs.stat(path, clb); // stat the target the link points to
595 596 597 598 599
		}

		return clb(null, lstat); // not a link, so the stat is already ok for us
	}

600
	private realPathIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error | null, realpath?: string) => void): void {
601 602 603 604 605
		if (lstat.isSymbolicLink()) {
			return fs.realpath(path, (error, realpath) => {
				if (error) {
					return clb(error);
				}
E
Erich Gamma 已提交
606

607 608 609
				return clb(null, realpath);
			});
		}
E
Erich Gamma 已提交
610

611
		return clb(null, path);
E
Erich Gamma 已提交
612
	}
613 614 615 616 617 618 619 620 621 622 623 624 625

	/**
	 * If we're searching for files in multiple workspace folders, then better prepend the
	 * name of the workspace folder to the path of the file. This way we'll be able to
	 * better filter files that are all on the top of a workspace folder and have all the
	 * same name. A typical example are `package.json` or `README.md` files.
	 */
	private getSearchPath(folderQuery: IFolderQuery, relativePath: string): string {
		if (folderQuery.folderName) {
			return path.join(folderQuery.folderName, relativePath);
		}
		return relativePath;
	}
E
Erich Gamma 已提交
626 627
}

628
export class Engine implements ISearchEngine<IRawFileMatch> {
629 630
	private folderQueries: IFolderQuery[];
	private extraFiles: URI[];
E
Erich Gamma 已提交
631 632
	private walker: FileWalker;

633
	constructor(config: IFileQuery) {
634
		this.folderQueries = config.folderQueries;
635
		this.extraFiles = config.extraFileResources || [];
636

E
Erich Gamma 已提交
637 638 639
		this.walker = new FileWalker(config);
	}

M
Matt Bierner 已提交
640 641
	search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgressMessage) => void, done: (error: Error | null, complete: ISearchEngineSuccess) => void): void {
		this.walker.walk(this.folderQueries, this.extraFiles, onResult, onProgress, (err: Error | null, isLimitHit: boolean) => {
C
chrmarti 已提交
642 643 644 645 646
			done(err, {
				limitHit: isLimitHit,
				stats: this.walker.getStats()
			});
		});
E
Erich Gamma 已提交
647 648
	}

R
Rob Lourens 已提交
649
	cancel(): void {
E
Erich Gamma 已提交
650 651
		this.walker.cancel();
	}
652 653 654 655 656 657 658 659
}

/**
 * This class exists to provide one interface on top of two ParsedExpressions, one for absolute expressions and one for relative expressions.
 * The absolute and relative expressions don't "have" to be kept separate, but this keeps us from having to path.join every single
 * file searched, it's only used for a text search with a searchPath
 */
class AbsoluteAndRelativeParsedExpression {
660 661
	private absoluteParsedExpr: glob.ParsedExpression | undefined;
	private relativeParsedExpr: glob.ParsedExpression | undefined;
662

C
Christof Marti 已提交
663 664
	constructor(public expression: glob.IExpression, private root: string) {
		this.init(expression);
665 666 667 668 669 670
	}

	/**
	 * Split the IExpression into its absolute and relative components, and glob.parse them separately.
	 */
	private init(expr: glob.IExpression): void {
671 672
		let absoluteGlobExpr: glob.IExpression | undefined;
		let relativeGlobExpr: glob.IExpression | undefined;
673 674 675 676 677
		Object.keys(expr)
			.filter(key => expr[key])
			.forEach(key => {
				if (path.isAbsolute(key)) {
					absoluteGlobExpr = absoluteGlobExpr || glob.getEmptyExpression();
678
					absoluteGlobExpr[key] = expr[key];
679 680
				} else {
					relativeGlobExpr = relativeGlobExpr || glob.getEmptyExpression();
681
					relativeGlobExpr[key] = expr[key];
682 683
				}
			});
684 685 686 687 688

		this.absoluteParsedExpr = absoluteGlobExpr && glob.parse(absoluteGlobExpr, { trimForExclusions: true });
		this.relativeParsedExpr = relativeGlobExpr && glob.parse(relativeGlobExpr, { trimForExclusions: true });
	}

R
Rob Lourens 已提交
689
	test(_path: string, basename?: string, hasSibling?: (name: string) => boolean | Promise<boolean>): string | Promise<string | null> | undefined | null {
690 691
		return (this.relativeParsedExpr && this.relativeParsedExpr(_path, basename, hasSibling)) ||
			(this.absoluteParsedExpr && this.absoluteParsedExpr(path.join(this.root, _path), basename, hasSibling));
692 693
	}

R
Rob Lourens 已提交
694
	getBasenameTerms(): string[] {
M
Matt Bierner 已提交
695
		const basenameTerms: string[] = [];
696 697 698 699 700 701 702 703 704 705 706
		if (this.absoluteParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.relativeParsedExpr));
		}

		return basenameTerms;
	}

R
Rob Lourens 已提交
707
	getPathTerms(): string[] {
M
Matt Bierner 已提交
708
		const pathTerms: string[] = [];
709 710 711 712 713 714 715 716 717 718
		if (this.absoluteParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.relativeParsedExpr));
		}

		return pathTerms;
	}
719
}
R
Rob Lourens 已提交
720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750

export function rgErrorMsgForDisplay(msg: string): string | undefined {
	const lines = msg.trim().split('\n');
	const firstLine = lines[0].trim();

	if (strings.startsWith(firstLine, 'Error parsing regex')) {
		return firstLine;
	}

	if (strings.startsWith(firstLine, 'regex parse error')) {
		return strings.uppercaseFirstLetter(lines[lines.length - 1].trim());
	}

	if (strings.startsWith(firstLine, 'error parsing glob') ||
		strings.startsWith(firstLine, 'unsupported encoding')) {
		// Uppercase first letter
		return firstLine.charAt(0).toUpperCase() + firstLine.substr(1);
	}

	if (firstLine === `Literal '\\n' not allowed.`) {
		// I won't localize this because none of the Ripgrep error messages are localized
		return `Literal '\\n' currently not supported`;
	}

	if (strings.startsWith(firstLine, 'Literal ')) {
		// Other unsupported chars
		return firstLine;
	}

	return undefined;
}