fileSearch.ts 24.3 KB
Newer Older
E
Erich Gamma 已提交
1 2 3 4 5
/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

C
Christof Marti 已提交
6
import * as childProcess from 'child_process';
7 8
import * as fs from 'fs';
import * as path from 'path';
B
Benjamin Pasero 已提交
9
import { Readable } from 'stream';
10
import { NodeStringDecoder, StringDecoder } from 'string_decoder';
11
import * as arrays from 'vs/base/common/arrays';
12 13 14 15 16
import { toErrorMessage } from 'vs/base/common/errorMessage';
import * as glob from 'vs/base/common/glob';
import * as normalization from 'vs/base/common/normalization';
import * as objects from 'vs/base/common/objects';
import { isEqualOrParent } from 'vs/base/common/paths';
17
import * as platform from 'vs/base/common/platform';
18
import { StopWatch } from 'vs/base/common/stopwatch';
19 20
import * as strings from 'vs/base/common/strings';
import * as types from 'vs/base/common/types';
21
import { URI } from 'vs/base/common/uri';
22 23
import * as extfs from 'vs/base/node/extfs';
import * as flow from 'vs/base/node/flow';
24
import { IFileQuery, IFolderQuery, IProgress, ISearchEngineStats } from 'vs/platform/search/common/search';
25
import { IRawFileMatch, ISearchEngine, ISearchEngineSuccess } from 'vs/workbench/services/search/node/search';
26
import { spawnRipgrepCmd } from './ripgrepFileSearch';
E
Erich Gamma 已提交
27

C
Christof Marti 已提交
28 29 30
enum Traversal {
	Node = 1,
	MacFind,
31 32
	LinuxFind,
	Ripgrep
C
Christof Marti 已提交
33 34 35
}

interface IDirectoryEntry {
36
	base: string;
C
Christof Marti 已提交
37 38 39 40 41 42 43 44 45
	relativePath: string;
	basename: string;
}

interface IDirectoryTree {
	rootEntries: IDirectoryEntry[];
	pathToEntries: { [relativePath: string]: IDirectoryEntry[] };
}

46 47 48 49 50
const killCmds = new Set<() => void>();
process.on('exit', () => {
	killCmds.forEach(cmd => cmd());
});

E
Erich Gamma 已提交
51
export class FileWalker {
52
	private config: IFileQuery;
C
Christof Marti 已提交
53
	private useRipgrep: boolean;
54
	private filePattern: string;
55
	private normalizedFilePatternLowercase: string;
56 57
	private includePattern: glob.ParsedExpression | undefined;
	private maxResults: number | null;
58
	private exists: boolean;
59
	private maxFilesize: number | null;
E
Erich Gamma 已提交
60 61 62
	private isLimitHit: boolean;
	private resultCount: number;
	private isCanceled: boolean;
63
	private fileWalkSW: StopWatch;
C
chrmarti 已提交
64 65
	private directoriesWalked: number;
	private filesWalked: number;
C
Christof Marti 已提交
66 67
	private traversal: Traversal;
	private errors: string[];
68
	private cmdSW: StopWatch;
C
Christof Marti 已提交
69
	private cmdResultCount: number;
E
Erich Gamma 已提交
70

71
	private folderExcludePatterns: Map<string, AbsoluteAndRelativeParsedExpression>;
72
	private globalExcludePattern: glob.ParsedExpression | undefined;
73

E
Erich Gamma 已提交
74 75
	private walkedPaths: { [path: string]: boolean; };

76
	constructor(config: IFileQuery, maxFileSize?: number) {
E
Erich Gamma 已提交
77
		this.config = config;
C
Christof Marti 已提交
78
		this.useRipgrep = config.useRipgrep !== false;
79
		this.filePattern = config.filePattern || '';
C
Christof Marti 已提交
80
		this.includePattern = config.includePattern && glob.parse(config.includePattern);
E
Erich Gamma 已提交
81
		this.maxResults = config.maxResults || null;
82
		this.exists = !!config.exists;
83
		this.maxFilesize = maxFileSize || null;
E
Erich Gamma 已提交
84
		this.walkedPaths = Object.create(null);
85 86
		this.resultCount = 0;
		this.isLimitHit = false;
C
chrmarti 已提交
87 88
		this.directoriesWalked = 0;
		this.filesWalked = 0;
C
Christof Marti 已提交
89 90
		this.traversal = Traversal.Node;
		this.errors = [];
B
Benjamin Pasero 已提交
91

92 93
		if (this.filePattern) {
			this.normalizedFilePatternLowercase = strings.stripWildcards(this.filePattern).toLowerCase();
B
Benjamin Pasero 已提交
94
		}
95 96

		this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);
97
		this.folderExcludePatterns = new Map<string, AbsoluteAndRelativeParsedExpression>();
98 99

		config.folderQueries.forEach(folderQuery => {
100
			const folderExcludeExpression: glob.IExpression = objects.assign({}, folderQuery.excludePattern || {}, this.config.excludePattern || {});
101 102

			// Add excludes for other root folders
103
			const fqPath = folderQuery.folder.fsPath;
104
			config.folderQueries
105 106
				.map(rootFolderQuery => rootFolderQuery.folder.fsPath)
				.filter(rootFolder => rootFolder !== fqPath)
107 108
				.forEach(otherRootFolder => {
					// Exclude nested root folders
109 110
					if (isEqualOrParent(otherRootFolder, fqPath)) {
						folderExcludeExpression[path.relative(fqPath, otherRootFolder)] = true;
111
					}
112 113
				});

114
			this.folderExcludePatterns.set(fqPath, new AbsoluteAndRelativeParsedExpression(folderExcludeExpression, fqPath));
115
		});
E
Erich Gamma 已提交
116 117 118 119 120 121
	}

	public cancel(): void {
		this.isCanceled = true;
	}

122
	public walk(folderQueries: IFolderQuery[], extraFiles: URI[], onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, done: (error: Error | null, isLimitHit: boolean) => void): void {
123
		this.fileWalkSW = StopWatch.create(false);
E
Erich Gamma 已提交
124

125
		// Support that the file pattern is a full path to a file that exists
126 127 128
		if (this.isCanceled) {
			return done(null, this.isLimitHit);
		}
E
Erich Gamma 已提交
129

130
		// For each extra file
131 132 133 134 135
		extraFiles.forEach(extraFilePath => {
			const basename = path.basename(extraFilePath.fsPath);
			if (this.globalExcludePattern && this.globalExcludePattern(extraFilePath.fsPath, basename)) {
				return; // excluded
			}
E
Erich Gamma 已提交
136

137 138 139
			// File: Check for match on file pattern and include pattern
			this.matchFile(onResult, { relativePath: extraFilePath.fsPath /* no workspace relative path */, basename });
		});
140

141 142 143 144 145 146 147 148
		let traverse = this.nodeJSTraversal;
		if (!this.maxFilesize) {
			if (this.useRipgrep) {
				this.traversal = Traversal.Ripgrep;
				traverse = this.cmdTraversal;
			} else if (platform.isMacintosh) {
				this.traversal = Traversal.MacFind;
				traverse = this.cmdTraversal;
R
Rob Lourens 已提交
149
			} else if (platform.isLinux) {
150 151
				this.traversal = Traversal.LinuxFind;
				traverse = this.cmdTraversal;
C
Christof Marti 已提交
152
			}
153
		}
C
Christof Marti 已提交
154

155 156
		const isNodeTraversal = traverse === this.nodeJSTraversal;
		if (!isNodeTraversal) {
157
			this.cmdSW = StopWatch.create(false);
158
		}
C
Christof Marti 已提交
159

160
		// For each root folder
161
		flow.parallel<IFolderQuery, void>(folderQueries, (folderQuery: IFolderQuery, rootFolderDone: (err: Error | null, result: void) => void) => {
162 163 164 165 166 167 168
			this.call(traverse, this, folderQuery, onResult, onMessage, (err?: Error) => {
				if (err) {
					const errorMessage = toErrorMessage(err);
					console.error(errorMessage);
					this.errors.push(errorMessage);
					rootFolderDone(err, undefined);
				} else {
169
					rootFolderDone(null, undefined);
170
				}
C
Christof Marti 已提交
171
			});
172
		}, (errors, result) => {
173
			this.fileWalkSW.stop();
M
Matt Bierner 已提交
174
			const err = errors ? arrays.coalesce(errors)[0] : null;
175
			done(err, this.isLimitHit);
C
Christof Marti 已提交
176 177
		});
	}
178

C
Christof Marti 已提交
179 180 181 182 183 184 185 186
	private call(fun: Function, that: any, ...args: any[]): void {
		try {
			fun.apply(that, args);
		} catch (e) {
			args[args.length - 1](e);
		}
	}

187 188
	private cmdTraversal(folderQuery: IFolderQuery, onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, cb: (err?: Error) => void): void {
		const rootFolder = folderQuery.folder.fsPath;
189
		const isMac = platform.isMacintosh;
190 191
		let cmd: childProcess.ChildProcess;
		const killCmd = () => cmd && cmd.kill();
192
		killCmds.add(killCmd);
193

194
		let done = (err?: Error) => {
195
			killCmds.delete(killCmd);
J
Johannes Rieken 已提交
196
			done = () => { };
197 198 199 200 201
			cb(err);
		};
		let leftover = '';
		let first = true;
		const tree = this.initDirectoryTree();
202

C
Christof Marti 已提交
203
		const useRipgrep = this.useRipgrep;
204 205
		let noSiblingsClauses: boolean;
		if (useRipgrep) {
206
			const ripgrep = spawnRipgrepCmd(this.config, folderQuery, this.config.includePattern, this.folderExcludePatterns.get(folderQuery.folder.fsPath).expression);
207 208
			cmd = ripgrep.cmd;
			noSiblingsClauses = !Object.keys(ripgrep.siblingClauses).length;
209

210 211 212 213 214 215 216 217 218
			const escapedArgs = ripgrep.rgArgs.args
				.map(arg => arg.match(/^-/) ? arg : `'${arg}'`)
				.join(' ');

			let rgCmd = `rg ${escapedArgs}\n - cwd: ${ripgrep.cwd}`;
			if (ripgrep.rgArgs.siblingClauses) {
				rgCmd += `\n - Sibling clauses: ${JSON.stringify(ripgrep.rgArgs.siblingClauses)}`;
			}
			onMessage({ message: rgCmd });
219 220 221 222
		} else {
			cmd = this.spawnFindCmd(folderQuery);
		}

223
		this.cmdResultCount = 0;
224
		this.collectStdout(cmd, 'utf8', useRipgrep, onMessage, (err: Error, stdout?: string, last?: boolean) => {
C
Christof Marti 已提交
225 226 227 228
			if (err) {
				done(err);
				return;
			}
229
			if (this.isLimitHit) {
230
				done();
231 232
				return;
			}
E
Erich Gamma 已提交
233

C
Christof Marti 已提交
234
			// Mac: uses NFD unicode form on disk, but we want NFC
235
			const normalized = leftover + (isMac ? normalization.normalizeNFC(stdout || '') : stdout);
236
			const relativeFiles = normalized.split(useRipgrep ? '\n' : '\n./');
C
Christof Marti 已提交
237
			if (!useRipgrep && first && normalized.length >= 2) {
238 239
				first = false;
				relativeFiles[0] = relativeFiles[0].trim().substr(2);
C
Christof Marti 已提交
240 241
			}

242 243 244 245 246 247 248
			if (last) {
				const n = relativeFiles.length;
				relativeFiles[n - 1] = relativeFiles[n - 1].trim();
				if (!relativeFiles[n - 1]) {
					relativeFiles.pop();
				}
			} else {
249
				leftover = relativeFiles.pop() || '';
C
Christof Marti 已提交
250 251
			}

252 253 254 255 256
			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
				done(new Error('Splitting up files failed'));
				return;
			}

257 258 259 260 261 262
			this.cmdResultCount += relativeFiles.length;

			if (useRipgrep && noSiblingsClauses) {
				for (const relativePath of relativeFiles) {
					const basename = path.basename(relativePath);
					this.matchFile(onResult, { base: rootFolder, relativePath, basename });
263 264 265 266
					if (this.isLimitHit) {
						killCmd();
						break;
					}
267
				}
268
				if (last || this.isLimitHit) {
269
					done();
270
				}
271

272 273 274
				return;
			}

275
			// TODO: Optimize siblings clauses with ripgrep here.
276
			this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
C
Christof Marti 已提交
277

278 279 280
			if (last) {
				this.matchDirectoryTree(tree, rootFolder, onResult);
				done();
281
			}
C
Christof Marti 已提交
282 283 284
		});
	}

285 286 287
	/**
	 * Public for testing.
	 */
288 289
	public spawnFindCmd(folderQuery: IFolderQuery) {
		const excludePattern = this.folderExcludePatterns.get(folderQuery.folder.fsPath);
290 291
		const basenames = excludePattern.getBasenameTerms();
		const pathTerms = excludePattern.getPathTerms();
292
		let args = ['-L', '.'];
293
		if (basenames.length || pathTerms.length) {
294
			args.push('-not', '(', '(');
295
			for (const basename of basenames) {
296
				args.push('-name', basename);
297 298
				args.push('-o');
			}
299
			for (const path of pathTerms) {
300
				args.push('-path', path);
301
				args.push('-o');
302
			}
303
			args.pop();
304 305 306
			args.push(')', '-prune', ')');
		}
		args.push('-type', 'f');
307
		return childProcess.spawn('find', args, { cwd: folderQuery.folder.fsPath });
308 309 310 311 312
	}

	/**
	 * Public for testing.
	 */
313
	public readStdout(cmd: childProcess.ChildProcess, encoding: string, isRipgrep: boolean, cb: (err: Error | null, stdout?: string) => void): void {
314
		let all = '';
315
		this.collectStdout(cmd, encoding, isRipgrep, () => { }, (err: Error, stdout?: string, last?: boolean) => {
316 317 318 319 320 321 322 323 324 325 326 327
			if (err) {
				cb(err);
				return;
			}

			all += stdout;
			if (last) {
				cb(null, all);
			}
		});
	}

328 329
	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, isRipgrep: boolean, onMessage: (message: IProgress) => void, cb: (err: Error | null, stdout?: string, last?: boolean) => void): void {
		let onData = (err: Error | null, stdout?: string, last?: boolean) => {
330
			if (err || last) {
331
				onData = () => { };
332 333 334 335

				if (this.cmdSW) {
					this.cmdSW.stop();
				}
336 337
			}
			cb(err, stdout, last);
C
Christof Marti 已提交
338 339
		};

340
		let gotData = false;
341 342 343
		if (cmd.stdout) {
			// Should be non-null, but #38195
			this.forwardData(cmd.stdout, encoding, onData);
344
			cmd.stdout.once('data', () => gotData = true);
345 346 347 348
		} else {
			onMessage({ message: 'stdout is null' });
		}

349 350 351 352 353 354 355
		let stderr: Buffer[];
		if (cmd.stderr) {
			// Should be non-null, but #38195
			stderr = this.collectData(cmd.stderr);
		} else {
			onMessage({ message: 'stderr is null' });
		}
356

357
		cmd.on('error', (err: Error) => {
358
			onData(err);
C
Christof Marti 已提交
359 360
		});

361
		cmd.on('close', (code: number) => {
362
			// ripgrep returns code=1 when no results are found
363 364
			let stderrText: string;
			if (isRipgrep ? (!gotData && (stderrText = this.decodeData(stderr, encoding)) && rgErrorMsgForDisplay(stderrText)) : code !== 0) {
365
				onData(new Error(`command failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
C
Christof Marti 已提交
366
			} else {
367 368 369
				if (isRipgrep && this.exists && code === 0) {
					this.isLimitHit = true;
				}
370
				onData(null, '', true);
C
Christof Marti 已提交
371 372 373 374
			}
		});
	}

375
	private forwardData(stream: Readable, encoding: string, cb: (err: Error | null, stdout?: string) => void): NodeStringDecoder {
376 377 378 379 380 381 382
		const decoder = new StringDecoder(encoding);
		stream.on('data', (data: Buffer) => {
			cb(null, decoder.write(data));
		});
		return decoder;
	}

C
Christof Marti 已提交
383
	private collectData(stream: Readable): Buffer[] {
384 385
		const buffers: Buffer[] = [];
		stream.on('data', (data: Buffer) => {
C
Christof Marti 已提交
386 387 388 389 390
			buffers.push(data);
		});
		return buffers;
	}

C
Christof Marti 已提交
391 392 393 394 395
	private decodeData(buffers: Buffer[], encoding: string): string {
		const decoder = new StringDecoder(encoding);
		return buffers.map(buffer => decoder.write(buffer)).join('');
	}

396 397 398 399 400 401 402 403 404
	private initDirectoryTree(): IDirectoryTree {
		const tree: IDirectoryTree = {
			rootEntries: [],
			pathToEntries: Object.create(null)
		};
		tree.pathToEntries['.'] = tree.rootEntries;
		return tree;
	}

405
	private addDirectoryEntries({ pathToEntries }: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
406 407
		// Support relative paths to files from a root resource (ignores excludes)
		if (relativeFiles.indexOf(this.filePattern) !== -1) {
408
			const basename = path.basename(this.filePattern);
409
			this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
C
Christof Marti 已提交
410 411
		}

412
		function add(relativePath: string) {
413 414
			const basename = path.basename(relativePath);
			const dirname = path.dirname(relativePath);
C
Christof Marti 已提交
415 416 417 418 419 420
			let entries = pathToEntries[dirname];
			if (!entries) {
				entries = pathToEntries[dirname] = [];
				add(dirname);
			}
			entries.push({
421
				base,
C
Christof Marti 已提交
422 423 424
				relativePath,
				basename
			});
425 426
		}
		relativeFiles.forEach(add);
C
Christof Marti 已提交
427 428
	}

429
	private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
430
		const self = this;
431
		const excludePattern = this.folderExcludePatterns.get(rootFolder);
C
Christof Marti 已提交
432 433 434
		const filePattern = this.filePattern;
		function matchDirectory(entries: IDirectoryEntry[]) {
			self.directoriesWalked++;
435
			const hasSibling = glob.hasSiblingFn(() => entries.map(entry => entry.basename));
C
Christof Marti 已提交
436
			for (let i = 0, n = entries.length; i < n; i++) {
437
				const entry = entries[i];
438
				const { relativePath, basename } = entry;
C
Christof Marti 已提交
439 440 441 442 443

				// Check exclude pattern
				// If the user searches for the exact file name, we adjust the glob matching
				// to ignore filtering by siblings because the user seems to know what she
				// is searching for and we want to include the result in that case anyway
444
				if (excludePattern.test(relativePath, basename, filePattern !== basename ? hasSibling : undefined)) {
C
Christof Marti 已提交
445 446
					continue;
				}
E
Erich Gamma 已提交
447

C
Christof Marti 已提交
448 449 450 451 452 453 454 455 456
				const sub = pathToEntries[relativePath];
				if (sub) {
					matchDirectory(sub);
				} else {
					self.filesWalked++;
					if (relativePath === filePattern) {
						continue; // ignore file if its path matches with the file pattern because that is already matched above
					}

457
					self.matchFile(onResult, entry);
C
Christof Marti 已提交
458
				}
459 460 461 462

				if (self.isLimitHit) {
					break;
				}
463
			}
C
Christof Marti 已提交
464 465 466 467
		}
		matchDirectory(rootEntries);
	}

468
	private nodeJSTraversal(folderQuery: IFolderQuery, onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, done: (err?: Error) => void): void {
C
Christof Marti 已提交
469
		this.directoriesWalked++;
470
		extfs.readdir(folderQuery.folder.fsPath, (error: Error, files: string[]) => {
C
Christof Marti 已提交
471 472 473 474
			if (error || this.isCanceled || this.isLimitHit) {
				return done();
			}

475 476 477
			if (this.isCanceled || this.isLimitHit) {
				return done();
			}
C
Christof Marti 已提交
478

479
			return this.doWalk(folderQuery, '', files, onResult, done);
480 481 482
		});
	}

483
	public getStats(): ISearchEngineStats {
C
chrmarti 已提交
484
		return {
R
Rob Lourens 已提交
485
			cmdTime: this.cmdSW && this.cmdSW.elapsed(),
486
			fileWalkTime: this.fileWalkSW.elapsed(),
C
Christof Marti 已提交
487
			traversal: Traversal[this.traversal],
C
chrmarti 已提交
488
			directoriesWalked: this.directoriesWalked,
489
			filesWalked: this.filesWalked,
C
Christof Marti 已提交
490
			cmdResultCount: this.cmdResultCount
C
chrmarti 已提交
491 492 493
		};
	}

494
	private doWalk(folderQuery: IFolderQuery, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error?: Error) => void): void {
495
		const rootFolder = folderQuery.folder;
E
Erich Gamma 已提交
496 497

		// Execute tasks on each file in parallel to optimize throughput
498
		const hasSibling = glob.hasSiblingFn(() => files);
499
		flow.parallel(files, (file: string, clb: (error: Error | null, _?: any) => void): void => {
E
Erich Gamma 已提交
500 501 502

			// Check canceled
			if (this.isCanceled || this.isLimitHit) {
503
				return clb(null);
E
Erich Gamma 已提交
504 505
			}

506
			// Check exclude pattern
E
Erich Gamma 已提交
507 508 509
			// If the user searches for the exact file name, we adjust the glob matching
			// to ignore filtering by siblings because the user seems to know what she
			// is searching for and we want to include the result in that case anyway
510
			let currentRelativePath = relativeParentPath ? [relativeParentPath, file].join(path.sep) : file;
511
			if (this.folderExcludePatterns.get(folderQuery.folder.fsPath).test(currentRelativePath, file, this.config.filePattern !== file ? hasSibling : undefined)) {
512
				return clb(null);
E
Erich Gamma 已提交
513 514
			}

515
			// Use lstat to detect links
516
			let currentAbsolutePath = [rootFolder.fsPath, currentRelativePath].join(path.sep);
517
			fs.lstat(currentAbsolutePath, (error, lstat) => {
518
				if (error || this.isCanceled || this.isLimitHit) {
519
					return clb(null);
520
				}
E
Erich Gamma 已提交
521

522
				// If the path is a link, we must instead use fs.stat() to find out if the
523 524 525 526
				// link is a directory or not because lstat will always return the stat of
				// the link which is always a file.
				this.statLinkIfNeeded(currentAbsolutePath, lstat, (error, stat) => {
					if (error || this.isCanceled || this.isLimitHit) {
527
						return clb(null);
528
					}
E
Erich Gamma 已提交
529

530 531
					// Directory: Follow directories
					if (stat.isDirectory()) {
C
chrmarti 已提交
532
						this.directoriesWalked++;
533

534 535
						// to really prevent loops with links we need to resolve the real path of them
						return this.realPathIfNeeded(currentAbsolutePath, lstat, (error, realpath) => {
536
							if (error || this.isCanceled || this.isLimitHit) {
537
								return clb(null);
538 539
							}

540
							realpath = realpath || '';
541
							if (this.walkedPaths[realpath]) {
542
								return clb(null); // escape when there are cycles (can happen with symlinks)
543
							}
E
Erich Gamma 已提交
544

545 546 547 548 549
							this.walkedPaths[realpath] = true; // remember as walked

							// Continue walking
							return extfs.readdir(currentAbsolutePath, (error: Error, children: string[]): void => {
								if (error || this.isCanceled || this.isLimitHit) {
550
									return clb(null);
551 552
								}

553
								this.doWalk(folderQuery, currentRelativePath, children, onResult, err => clb(err || null));
554 555
							});
						});
556
					}
E
Erich Gamma 已提交
557

558 559
					// File: Check for match on file pattern and include pattern
					else {
C
chrmarti 已提交
560
						this.filesWalked++;
C
Christof Marti 已提交
561
						if (currentRelativePath === this.filePattern) {
562
							return clb(null, undefined); // ignore file if its path matches with the file pattern because checkFilePatternRelativeMatch() takes care of those
563
						}
E
Erich Gamma 已提交
564

565
						if (this.maxFilesize && types.isNumber(stat.size) && stat.size > this.maxFilesize) {
566
							return clb(null, undefined); // ignore file if max file size is hit
567 568
						}

569
						this.matchFile(onResult, { base: rootFolder.fsPath, relativePath: currentRelativePath, basename: file, size: stat.size });
570 571 572
					}

					// Unwind
573
					return clb(null, undefined);
574
				});
E
Erich Gamma 已提交
575 576 577 578 579 580
			});
		}, (error: Error[]): void => {
			if (error) {
				error = arrays.coalesce(error); // find any error by removing null values first
			}

581
			return done(error && error.length > 0 ? error[0] : undefined);
E
Erich Gamma 已提交
582 583 584
		});
	}

585 586
	private matchFile(onResult: (result: IRawFileMatch) => void, candidate: IRawFileMatch): void {
		if (this.isFilePatternMatch(candidate.relativePath) && (!this.includePattern || this.includePattern(candidate.relativePath, candidate.basename))) {
587 588
			this.resultCount++;

589
			if (this.exists || (this.maxResults && this.resultCount > this.maxResults)) {
590 591 592 593
				this.isLimitHit = true;
			}

			if (!this.isLimitHit) {
594
				onResult(candidate);
595 596 597 598
			}
		}
	}

B
polish  
Benjamin Pasero 已提交
599
	private isFilePatternMatch(path: string): boolean {
600 601 602

		// Check for search pattern
		if (this.filePattern) {
603 604 605 606
			if (this.filePattern === '*') {
				return true; // support the all-matching wildcard
			}

607
			return strings.fuzzyContains(path, this.normalizedFilePatternLowercase);
608 609 610 611 612 613
		}

		// No patterns means we match all
		return true;
	}

614
	private statLinkIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error | null, stat: fs.Stats) => void): void {
615
		if (lstat.isSymbolicLink()) {
616
			return fs.stat(path, clb); // stat the target the link points to
617 618 619 620 621
		}

		return clb(null, lstat); // not a link, so the stat is already ok for us
	}

622
	private realPathIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error | null, realpath?: string) => void): void {
623 624 625 626 627
		if (lstat.isSymbolicLink()) {
			return fs.realpath(path, (error, realpath) => {
				if (error) {
					return clb(error);
				}
E
Erich Gamma 已提交
628

629 630 631
				return clb(null, realpath);
			});
		}
E
Erich Gamma 已提交
632

633
		return clb(null, path);
E
Erich Gamma 已提交
634 635 636
	}
}

637
export class Engine implements ISearchEngine<IRawFileMatch> {
638 639
	private folderQueries: IFolderQuery[];
	private extraFiles: URI[];
E
Erich Gamma 已提交
640 641
	private walker: FileWalker;

642
	constructor(config: IFileQuery) {
643
		this.folderQueries = config.folderQueries;
644
		this.extraFiles = config.extraFileResources || [];
645

E
Erich Gamma 已提交
646 647 648
		this.walker = new FileWalker(config);
	}

649
	public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISearchEngineSuccess) => void): void {
650
		this.walker.walk(this.folderQueries, this.extraFiles, onResult, onProgress, (err: Error, isLimitHit: boolean) => {
C
chrmarti 已提交
651 652 653 654 655
			done(err, {
				limitHit: isLimitHit,
				stats: this.walker.getStats()
			});
		});
E
Erich Gamma 已提交
656 657 658 659 660
	}

	public cancel(): void {
		this.walker.cancel();
	}
661 662 663 664 665 666 667 668
}

/**
 * This class exists to provide one interface on top of two ParsedExpressions, one for absolute expressions and one for relative expressions.
 * The absolute and relative expressions don't "have" to be kept separate, but this keeps us from having to path.join every single
 * file searched, it's only used for a text search with a searchPath
 */
class AbsoluteAndRelativeParsedExpression {
669 670
	private absoluteParsedExpr: glob.ParsedExpression | undefined;
	private relativeParsedExpr: glob.ParsedExpression | undefined;
671

C
Christof Marti 已提交
672 673
	constructor(public expression: glob.IExpression, private root: string) {
		this.init(expression);
674 675 676 677 678 679
	}

	/**
	 * Split the IExpression into its absolute and relative components, and glob.parse them separately.
	 */
	private init(expr: glob.IExpression): void {
680 681
		let absoluteGlobExpr: glob.IExpression | undefined;
		let relativeGlobExpr: glob.IExpression | undefined;
682 683 684 685 686
		Object.keys(expr)
			.filter(key => expr[key])
			.forEach(key => {
				if (path.isAbsolute(key)) {
					absoluteGlobExpr = absoluteGlobExpr || glob.getEmptyExpression();
687
					absoluteGlobExpr[key] = expr[key];
688 689
				} else {
					relativeGlobExpr = relativeGlobExpr || glob.getEmptyExpression();
690
					relativeGlobExpr[key] = expr[key];
691 692
				}
			});
693 694 695 696 697

		this.absoluteParsedExpr = absoluteGlobExpr && glob.parse(absoluteGlobExpr, { trimForExclusions: true });
		this.relativeParsedExpr = relativeGlobExpr && glob.parse(relativeGlobExpr, { trimForExclusions: true });
	}

698
	public test(_path: string, basename?: string, hasSibling?: (name: string) => boolean | Promise<boolean>): string | Promise<string | null> | undefined | null {
699 700
		return (this.relativeParsedExpr && this.relativeParsedExpr(_path, basename, hasSibling)) ||
			(this.absoluteParsedExpr && this.absoluteParsedExpr(path.join(this.root, _path), basename, hasSibling));
701 702 703
	}

	public getBasenameTerms(): string[] {
M
Matt Bierner 已提交
704
		const basenameTerms: string[] = [];
705 706 707 708 709 710 711 712 713 714 715 716
		if (this.absoluteParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.relativeParsedExpr));
		}

		return basenameTerms;
	}

	public getPathTerms(): string[] {
M
Matt Bierner 已提交
717
		const pathTerms: string[] = [];
718 719 720 721 722 723 724 725 726 727
		if (this.absoluteParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.relativeParsedExpr));
		}

		return pathTerms;
	}
728
}
R
Rob Lourens 已提交
729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759

export function rgErrorMsgForDisplay(msg: string): string | undefined {
	const lines = msg.trim().split('\n');
	const firstLine = lines[0].trim();

	if (strings.startsWith(firstLine, 'Error parsing regex')) {
		return firstLine;
	}

	if (strings.startsWith(firstLine, 'regex parse error')) {
		return strings.uppercaseFirstLetter(lines[lines.length - 1].trim());
	}

	if (strings.startsWith(firstLine, 'error parsing glob') ||
		strings.startsWith(firstLine, 'unsupported encoding')) {
		// Uppercase first letter
		return firstLine.charAt(0).toUpperCase() + firstLine.substr(1);
	}

	if (firstLine === `Literal '\\n' not allowed.`) {
		// I won't localize this because none of the Ripgrep error messages are localized
		return `Literal '\\n' currently not supported`;
	}

	if (strings.startsWith(firstLine, 'Literal ')) {
		// Other unsupported chars
		return firstLine;
	}

	return undefined;
}