fileSearch.ts 24.4 KB
Newer Older
E
Erich Gamma 已提交
1 2 3 4 5 6 7
/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

'use strict';

C
Christof Marti 已提交
8
import * as childProcess from 'child_process';
J
Johannes Rieken 已提交
9 10
import { StringDecoder, NodeStringDecoder } from 'string_decoder';
import { toErrorMessage } from 'vs/base/common/errorMessage';
11 12
import * as fs from 'fs';
import * as path from 'path';
13
import { isEqualOrParent } from 'vs/base/common/paths';
B
Benjamin Pasero 已提交
14
import { Readable } from 'stream';
15
import { TPromise } from 'vs/base/common/winjs.base';
E
Erich Gamma 已提交
16

17 18 19 20
import * as objects from 'vs/base/common/objects';
import * as arrays from 'vs/base/common/arrays';
import * as platform from 'vs/base/common/platform';
import * as strings from 'vs/base/common/strings';
21
import * as normalization from 'vs/base/common/normalization';
22 23
import * as types from 'vs/base/common/types';
import * as glob from 'vs/base/common/glob';
J
Johannes Rieken 已提交
24
import { IProgress, IUncachedSearchStats } from 'vs/platform/search/common/search';
E
Erich Gamma 已提交
25

26 27
import * as extfs from 'vs/base/node/extfs';
import * as flow from 'vs/base/node/flow';
28
import { IRawFileMatch, IRawSearch, ISearchEngine, IFolderSearch, ISerializedSearchSuccess } from './search';
29
import { spawnRipgrepCmd } from './ripgrepFileSearch';
30
import { rgErrorMsgForDisplay } from './ripgrepTextSearch';
E
Erich Gamma 已提交
31

C
Christof Marti 已提交
32 33 34
enum Traversal {
	Node = 1,
	MacFind,
C
Christof Marti 已提交
35
	WindowsDir,
36 37
	LinuxFind,
	Ripgrep
C
Christof Marti 已提交
38 39 40
}

interface IDirectoryEntry {
41
	base: string;
C
Christof Marti 已提交
42 43 44 45 46 47 48 49 50
	relativePath: string;
	basename: string;
}

interface IDirectoryTree {
	rootEntries: IDirectoryEntry[];
	pathToEntries: { [relativePath: string]: IDirectoryEntry[] };
}

E
Erich Gamma 已提交
51 52
export class FileWalker {
	private config: IRawSearch;
C
Christof Marti 已提交
53
	private useRipgrep: boolean;
54
	private filePattern: string;
55
	private normalizedFilePatternLowercase: string;
C
Christof Marti 已提交
56
	private includePattern: glob.ParsedExpression;
E
Erich Gamma 已提交
57
	private maxResults: number;
58
	private exists: boolean;
59
	private maxFilesize: number;
E
Erich Gamma 已提交
60 61 62
	private isLimitHit: boolean;
	private resultCount: number;
	private isCanceled: boolean;
C
chrmarti 已提交
63 64 65
	private fileWalkStartTime: number;
	private directoriesWalked: number;
	private filesWalked: number;
C
Christof Marti 已提交
66 67 68 69 70
	private traversal: Traversal;
	private errors: string[];
	private cmdForkStartTime: number;
	private cmdForkResultTime: number;
	private cmdResultCount: number;
E
Erich Gamma 已提交
71

72
	private folderExcludePatterns: Map<string, AbsoluteAndRelativeParsedExpression>;
73 74
	private globalExcludePattern: glob.ParsedExpression;

E
Erich Gamma 已提交
75 76 77 78
	private walkedPaths: { [path: string]: boolean; };

	constructor(config: IRawSearch) {
		this.config = config;
C
Christof Marti 已提交
79
		this.useRipgrep = config.useRipgrep !== false;
80
		this.filePattern = config.filePattern;
C
Christof Marti 已提交
81
		this.includePattern = config.includePattern && glob.parse(config.includePattern);
E
Erich Gamma 已提交
82
		this.maxResults = config.maxResults || null;
83
		this.exists = config.exists;
84
		this.maxFilesize = config.maxFilesize || null;
E
Erich Gamma 已提交
85
		this.walkedPaths = Object.create(null);
86 87
		this.resultCount = 0;
		this.isLimitHit = false;
C
chrmarti 已提交
88 89
		this.directoriesWalked = 0;
		this.filesWalked = 0;
C
Christof Marti 已提交
90 91
		this.traversal = Traversal.Node;
		this.errors = [];
B
Benjamin Pasero 已提交
92

93 94
		if (this.filePattern) {
			this.normalizedFilePatternLowercase = strings.stripWildcards(this.filePattern).toLowerCase();
B
Benjamin Pasero 已提交
95
		}
96 97

		this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);
98
		this.folderExcludePatterns = new Map<string, AbsoluteAndRelativeParsedExpression>();
99 100

		config.folderQueries.forEach(folderQuery => {
101
			const folderExcludeExpression: glob.IExpression = objects.assign({}, folderQuery.excludePattern || {}, this.config.excludePattern || {});
102 103 104 105 106

			// Add excludes for other root folders
			config.folderQueries
				.map(rootFolderQuery => rootFolderQuery.folder)
				.filter(rootFolder => rootFolder !== folderQuery.folder)
107 108 109 110 111
				.forEach(otherRootFolder => {
					// Exclude nested root folders
					if (isEqualOrParent(otherRootFolder, folderQuery.folder)) {
						folderExcludeExpression[path.relative(folderQuery.folder, otherRootFolder)] = true;
					}
112 113
				});

114
			this.folderExcludePatterns.set(folderQuery.folder, new AbsoluteAndRelativeParsedExpression(folderExcludeExpression, folderQuery.folder));
115
		});
E
Erich Gamma 已提交
116 117 118 119 120 121
	}

	public cancel(): void {
		this.isCanceled = true;
	}

122
	public walk(folderQueries: IFolderSearch[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, done: (error: Error, isLimitHit: boolean) => void): void {
C
chrmarti 已提交
123
		this.fileWalkStartTime = Date.now();
E
Erich Gamma 已提交
124

125
		// Support that the file pattern is a full path to a file that exists
126 127 128
		if (this.isCanceled) {
			return done(null, this.isLimitHit);
		}
E
Erich Gamma 已提交
129

130 131 132 133 134 135 136
		// For each extra file
		if (extraFiles) {
			extraFiles.forEach(extraFilePath => {
				const basename = path.basename(extraFilePath);
				if (this.globalExcludePattern && this.globalExcludePattern(extraFilePath, basename)) {
					return; // excluded
				}
E
Erich Gamma 已提交
137

138 139 140 141
				// File: Check for match on file pattern and include pattern
				this.matchFile(onResult, { relativePath: extraFilePath /* no workspace relative path */, basename });
			});
		}
142

143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
		let traverse = this.nodeJSTraversal;
		if (!this.maxFilesize) {
			if (this.useRipgrep) {
				this.traversal = Traversal.Ripgrep;
				traverse = this.cmdTraversal;
			} else if (platform.isMacintosh) {
				this.traversal = Traversal.MacFind;
				traverse = this.cmdTraversal;
				// Disable 'dir' for now (#11181, #11179, #11183, #11182).
			} /* else if (platform.isWindows) {
				this.traversal = Traversal.WindowsDir;
				traverse = this.windowsDirTraversal;
			} */ else if (platform.isLinux) {
				this.traversal = Traversal.LinuxFind;
				traverse = this.cmdTraversal;
C
Christof Marti 已提交
158
			}
159
		}
C
Christof Marti 已提交
160

161 162 163 164
		const isNodeTraversal = traverse === this.nodeJSTraversal;
		if (!isNodeTraversal) {
			this.cmdForkStartTime = Date.now();
		}
C
Christof Marti 已提交
165

166 167 168 169 170 171 172 173 174 175 176
		// For each root folder
		flow.parallel<IFolderSearch, void>(folderQueries, (folderQuery: IFolderSearch, rootFolderDone: (err: Error, result: void) => void) => {
			this.call(traverse, this, folderQuery, onResult, onMessage, (err?: Error) => {
				if (err) {
					const errorMessage = toErrorMessage(err);
					console.error(errorMessage);
					this.errors.push(errorMessage);
					rootFolderDone(err, undefined);
				} else {
					rootFolderDone(undefined, undefined);
				}
C
Christof Marti 已提交
177
			});
178 179 180
		}, (errors, result) => {
			const err = errors ? errors.filter(e => !!e)[0] : null;
			done(err, this.isLimitHit);
C
Christof Marti 已提交
181 182
		});
	}
183

C
Christof Marti 已提交
184 185 186 187 188 189 190 191
	private call(fun: Function, that: any, ...args: any[]): void {
		try {
			fun.apply(that, args);
		} catch (e) {
			args[args.length - 1](e);
		}
	}

192
	private cmdTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, cb: (err?: Error) => void): void {
193
		const rootFolder = folderQuery.folder;
194
		const isMac = platform.isMacintosh;
195 196 197
		let cmd: childProcess.ChildProcess;
		const killCmd = () => cmd && cmd.kill();

198
		let done = (err?: Error) => {
199
			process.removeListener('exit', killCmd);
J
Johannes Rieken 已提交
200
			done = () => { };
201 202 203 204 205
			cb(err);
		};
		let leftover = '';
		let first = true;
		const tree = this.initDirectoryTree();
206

C
Christof Marti 已提交
207
		const useRipgrep = this.useRipgrep;
208 209
		let noSiblingsClauses: boolean;
		if (useRipgrep) {
210
			const ripgrep = spawnRipgrepCmd(this.config, folderQuery, this.config.includePattern, this.folderExcludePatterns.get(folderQuery.folder).expression);
211 212
			cmd = ripgrep.cmd;
			noSiblingsClauses = !Object.keys(ripgrep.siblingClauses).length;
213 214 215 216 217 218 219 220 221 222 223 224

			process.nextTick(() => {
				const escapedArgs = ripgrep.rgArgs.args
					.map(arg => arg.match(/^-/) ? arg : `'${arg}'`)
					.join(' ');

				let rgCmd = `rg ${escapedArgs}\n - cwd: ${ripgrep.cwd}`;
				if (ripgrep.rgArgs.siblingClauses) {
					rgCmd += `\n - Sibling clauses: ${JSON.stringify(ripgrep.rgArgs.siblingClauses)}`;
				}
				onMessage({ message: rgCmd });
			});
225 226 227 228
		} else {
			cmd = this.spawnFindCmd(folderQuery);
		}

229
		process.on('exit', killCmd);
230
		this.collectStdout(cmd, 'utf8', useRipgrep, onMessage, (err: Error, stdout?: string, last?: boolean) => {
C
Christof Marti 已提交
231 232 233 234
			if (err) {
				done(err);
				return;
			}
235
			if (this.isLimitHit) {
236
				done();
237 238
				return;
			}
E
Erich Gamma 已提交
239

C
Christof Marti 已提交
240
			// Mac: uses NFD unicode form on disk, but we want NFC
241
			const normalized = leftover + (isMac ? normalization.normalizeNFC(stdout) : stdout);
242
			const relativeFiles = normalized.split(useRipgrep ? '\n' : '\n./');
C
Christof Marti 已提交
243
			if (!useRipgrep && first && normalized.length >= 2) {
244 245
				first = false;
				relativeFiles[0] = relativeFiles[0].trim().substr(2);
C
Christof Marti 已提交
246 247
			}

248 249 250 251 252 253 254 255
			if (last) {
				const n = relativeFiles.length;
				relativeFiles[n - 1] = relativeFiles[n - 1].trim();
				if (!relativeFiles[n - 1]) {
					relativeFiles.pop();
				}
			} else {
				leftover = relativeFiles.pop();
C
Christof Marti 已提交
256 257
			}

258 259 260 261 262
			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
				done(new Error('Splitting up files failed'));
				return;
			}

263 264 265 266 267 268
			this.cmdResultCount += relativeFiles.length;

			if (useRipgrep && noSiblingsClauses) {
				for (const relativePath of relativeFiles) {
					const basename = path.basename(relativePath);
					this.matchFile(onResult, { base: rootFolder, relativePath, basename });
269 270 271 272
					if (this.isLimitHit) {
						killCmd();
						break;
					}
273
				}
274
				if (last || this.isLimitHit) {
275
					done();
276
				}
277

278 279 280
				return;
			}

281
			// TODO: Optimize siblings clauses with ripgrep here.
282
			this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
C
Christof Marti 已提交
283

284 285 286
			if (last) {
				this.matchDirectoryTree(tree, rootFolder, onResult);
				done();
287
			}
C
Christof Marti 已提交
288 289 290
		});
	}

291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
	// protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
	// 	const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
	// 	this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
	// 		if (err) {
	// 			done(err);
	// 			return;
	// 		}

	// 		const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
	// 		relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
	// 		const n = relativeFiles.length;
	// 		relativeFiles[n - 1] = relativeFiles[n - 1].trim();
	// 		if (!relativeFiles[n - 1]) {
	// 			relativeFiles.pop();
	// 		}

	// 		if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
	// 			done(new Error('Splitting up files failed'));
	// 			return;
	// 		}

	// 		this.matchFiles(rootFolder, relativeFiles, onResult);

	// 		done();
	// 	});
	// }

318 319 320
	/**
	 * Public for testing.
	 */
321
	public spawnFindCmd(folderQuery: IFolderSearch) {
322
		const excludePattern = this.folderExcludePatterns.get(folderQuery.folder);
323 324
		const basenames = excludePattern.getBasenameTerms();
		const pathTerms = excludePattern.getPathTerms();
325
		let args = ['-L', '.'];
326
		if (basenames.length || pathTerms.length) {
327
			args.push('-not', '(', '(');
328
			for (const basename of basenames) {
329
				args.push('-name', basename);
330 331
				args.push('-o');
			}
332
			for (const path of pathTerms) {
333
				args.push('-path', path);
334
				args.push('-o');
335
			}
336
			args.pop();
337 338 339
			args.push(')', '-prune', ')');
		}
		args.push('-type', 'f');
340
		return childProcess.spawn('find', args, { cwd: folderQuery.folder });
341 342 343 344 345
	}

	/**
	 * Public for testing.
	 */
346
	public readStdout(cmd: childProcess.ChildProcess, encoding: string, isRipgrep: boolean, cb: (err: Error, stdout?: string) => void): void {
347
		let all = '';
348
		this.collectStdout(cmd, encoding, isRipgrep, () => { }, (err: Error, stdout?: string, last?: boolean) => {
349 350 351 352 353 354 355 356 357 358 359 360
			if (err) {
				cb(err);
				return;
			}

			all += stdout;
			if (last) {
				cb(null, all);
			}
		});
	}

361 362
	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, isRipgrep: boolean, onMessage: (message: IProgress) => void, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
		let onData = (err: Error, stdout?: string, last?: boolean) => {
363
			if (err || last) {
364
				onData = () => { };
365 366 367
				this.cmdForkResultTime = Date.now();
			}
			cb(err, stdout, last);
C
Christof Marti 已提交
368 369
		};

370
		let gotData = false;
371 372 373
		if (cmd.stdout) {
			// Should be non-null, but #38195
			this.forwardData(cmd.stdout, encoding, onData);
374
			cmd.stdout.once('data', () => gotData = true);
375 376 377 378
		} else {
			onMessage({ message: 'stdout is null' });
		}

379 380 381 382 383 384 385
		let stderr: Buffer[];
		if (cmd.stderr) {
			// Should be non-null, but #38195
			stderr = this.collectData(cmd.stderr);
		} else {
			onMessage({ message: 'stderr is null' });
		}
386

387
		cmd.on('error', (err: Error) => {
388
			onData(err);
C
Christof Marti 已提交
389 390
		});

391
		cmd.on('close', (code: number) => {
392
			// ripgrep returns code=1 when no results are found
393 394
			let stderrText, displayMsg: string;
			if (isRipgrep ? (!gotData && (stderrText = this.decodeData(stderr, encoding)) && (displayMsg = rgErrorMsgForDisplay(stderrText))) : code !== 0) {
395
				onData(new Error(`command failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
C
Christof Marti 已提交
396
			} else {
397 398 399
				if (isRipgrep && this.exists && code === 0) {
					this.isLimitHit = true;
				}
400
				onData(null, '', true);
C
Christof Marti 已提交
401 402 403 404
			}
		});
	}

405 406 407 408 409 410 411 412
	private forwardData(stream: Readable, encoding: string, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
		const decoder = new StringDecoder(encoding);
		stream.on('data', (data: Buffer) => {
			cb(null, decoder.write(data));
		});
		return decoder;
	}

C
Christof Marti 已提交
413
	private collectData(stream: Readable): Buffer[] {
414 415
		const buffers: Buffer[] = [];
		stream.on('data', (data: Buffer) => {
C
Christof Marti 已提交
416 417 418 419 420
			buffers.push(data);
		});
		return buffers;
	}

C
Christof Marti 已提交
421 422 423 424 425
	private decodeData(buffers: Buffer[], encoding: string): string {
		const decoder = new StringDecoder(encoding);
		return buffers.map(buffer => decoder.write(buffer)).join('');
	}

426 427 428 429 430 431 432 433 434
	private initDirectoryTree(): IDirectoryTree {
		const tree: IDirectoryTree = {
			rootEntries: [],
			pathToEntries: Object.create(null)
		};
		tree.pathToEntries['.'] = tree.rootEntries;
		return tree;
	}

435
	private addDirectoryEntries({ pathToEntries }: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
436 437
		// Support relative paths to files from a root resource (ignores excludes)
		if (relativeFiles.indexOf(this.filePattern) !== -1) {
438
			const basename = path.basename(this.filePattern);
439
			this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
C
Christof Marti 已提交
440 441
		}

442
		function add(relativePath: string) {
443 444
			const basename = path.basename(relativePath);
			const dirname = path.dirname(relativePath);
C
Christof Marti 已提交
445 446 447 448 449 450
			let entries = pathToEntries[dirname];
			if (!entries) {
				entries = pathToEntries[dirname] = [];
				add(dirname);
			}
			entries.push({
451
				base,
C
Christof Marti 已提交
452 453 454
				relativePath,
				basename
			});
455 456
		}
		relativeFiles.forEach(add);
C
Christof Marti 已提交
457 458
	}

459
	private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
460
		const self = this;
461
		const excludePattern = this.folderExcludePatterns.get(rootFolder);
C
Christof Marti 已提交
462 463 464 465
		const filePattern = this.filePattern;
		function matchDirectory(entries: IDirectoryEntry[]) {
			self.directoriesWalked++;
			for (let i = 0, n = entries.length; i < n; i++) {
466
				const entry = entries[i];
467
				const { relativePath, basename } = entry;
C
Christof Marti 已提交
468 469 470 471 472

				// Check exclude pattern
				// If the user searches for the exact file name, we adjust the glob matching
				// to ignore filtering by siblings because the user seems to know what she
				// is searching for and we want to include the result in that case anyway
473
				if (excludePattern.test(relativePath, basename, () => filePattern !== basename ? entries.map(entry => entry.basename) : [])) {
C
Christof Marti 已提交
474 475
					continue;
				}
E
Erich Gamma 已提交
476

C
Christof Marti 已提交
477 478 479 480 481 482 483 484 485
				const sub = pathToEntries[relativePath];
				if (sub) {
					matchDirectory(sub);
				} else {
					self.filesWalked++;
					if (relativePath === filePattern) {
						continue; // ignore file if its path matches with the file pattern because that is already matched above
					}

486
					self.matchFile(onResult, entry);
C
Christof Marti 已提交
487
				}
488 489 490 491

				if (self.isLimitHit) {
					break;
				}
492
			}
C
Christof Marti 已提交
493 494 495 496
		}
		matchDirectory(rootEntries);
	}

497
	private nodeJSTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, done: (err?: Error) => void): void {
C
Christof Marti 已提交
498
		this.directoriesWalked++;
499
		extfs.readdir(folderQuery.folder, (error: Error, files: string[]) => {
C
Christof Marti 已提交
500 501 502 503
			if (error || this.isCanceled || this.isLimitHit) {
				return done();
			}

504 505 506
			if (this.isCanceled || this.isLimitHit) {
				return done();
			}
C
Christof Marti 已提交
507

508
			return this.doWalk(folderQuery, '', files, onResult, done);
509 510 511
		});
	}

512
	public getStats(): IUncachedSearchStats {
C
chrmarti 已提交
513
		return {
514
			fromCache: false,
C
Christof Marti 已提交
515 516
			traversal: Traversal[this.traversal],
			errors: this.errors,
C
chrmarti 已提交
517 518 519
			fileWalkStartTime: this.fileWalkStartTime,
			fileWalkResultTime: Date.now(),
			directoriesWalked: this.directoriesWalked,
520
			filesWalked: this.filesWalked,
C
Christof Marti 已提交
521 522 523 524
			resultCount: this.resultCount,
			cmdForkStartTime: this.cmdForkStartTime,
			cmdForkResultTime: this.cmdForkResultTime,
			cmdResultCount: this.cmdResultCount
C
chrmarti 已提交
525 526 527
		};
	}

528 529
	private doWalk(folderQuery: IFolderSearch, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error) => void): void {
		const rootFolder = folderQuery.folder;
E
Erich Gamma 已提交
530 531

		// Execute tasks on each file in parallel to optimize throughput
532
		flow.parallel(files, (file: string, clb: (error: Error, result: {}) => void): void => {
E
Erich Gamma 已提交
533 534 535

			// Check canceled
			if (this.isCanceled || this.isLimitHit) {
536
				return clb(null, undefined);
E
Erich Gamma 已提交
537 538 539 540 541 542
			}

			// If the user searches for the exact file name, we adjust the glob matching
			// to ignore filtering by siblings because the user seems to know what she
			// is searching for and we want to include the result in that case anyway
			let siblings = files;
543
			if (this.config.filePattern === file) {
E
Erich Gamma 已提交
544 545 546 547
				siblings = [];
			}

			// Check exclude pattern
548
			let currentRelativePath = relativeParentPath ? [relativeParentPath, file].join(path.sep) : file;
549
			if (this.folderExcludePatterns.get(folderQuery.folder).test(currentRelativePath, file, () => siblings)) {
550
				return clb(null, undefined);
E
Erich Gamma 已提交
551 552
			}

553
			// Use lstat to detect links
554
			let currentAbsolutePath = [rootFolder, currentRelativePath].join(path.sep);
555
			fs.lstat(currentAbsolutePath, (error, lstat) => {
556
				if (error || this.isCanceled || this.isLimitHit) {
557
					return clb(null, undefined);
558
				}
E
Erich Gamma 已提交
559

560
				// If the path is a link, we must instead use fs.stat() to find out if the
561 562 563 564
				// link is a directory or not because lstat will always return the stat of
				// the link which is always a file.
				this.statLinkIfNeeded(currentAbsolutePath, lstat, (error, stat) => {
					if (error || this.isCanceled || this.isLimitHit) {
565
						return clb(null, undefined);
566
					}
E
Erich Gamma 已提交
567

568 569
					// Directory: Follow directories
					if (stat.isDirectory()) {
C
chrmarti 已提交
570
						this.directoriesWalked++;
571

572 573
						// to really prevent loops with links we need to resolve the real path of them
						return this.realPathIfNeeded(currentAbsolutePath, lstat, (error, realpath) => {
574
							if (error || this.isCanceled || this.isLimitHit) {
575
								return clb(null, undefined);
576 577
							}

578
							if (this.walkedPaths[realpath]) {
579
								return clb(null, undefined); // escape when there are cycles (can happen with symlinks)
580
							}
E
Erich Gamma 已提交
581

582 583 584 585 586
							this.walkedPaths[realpath] = true; // remember as walked

							// Continue walking
							return extfs.readdir(currentAbsolutePath, (error: Error, children: string[]): void => {
								if (error || this.isCanceled || this.isLimitHit) {
587
									return clb(null, undefined);
588 589
								}

590
								this.doWalk(folderQuery, currentRelativePath, children, onResult, err => clb(err, undefined));
591 592
							});
						});
593
					}
E
Erich Gamma 已提交
594

595 596
					// File: Check for match on file pattern and include pattern
					else {
C
chrmarti 已提交
597
						this.filesWalked++;
C
Christof Marti 已提交
598
						if (currentRelativePath === this.filePattern) {
599
							return clb(null, undefined); // ignore file if its path matches with the file pattern because checkFilePatternRelativeMatch() takes care of those
600
						}
E
Erich Gamma 已提交
601

602
						if (this.maxFilesize && types.isNumber(stat.size) && stat.size > this.maxFilesize) {
603
							return clb(null, undefined); // ignore file if max file size is hit
604 605
						}

606
						this.matchFile(onResult, { base: rootFolder, relativePath: currentRelativePath, basename: file, size: stat.size });
607 608 609
					}

					// Unwind
610
					return clb(null, undefined);
611
				});
E
Erich Gamma 已提交
612 613 614 615 616 617
			});
		}, (error: Error[]): void => {
			if (error) {
				error = arrays.coalesce(error); // find any error by removing null values first
			}

C
Christof Marti 已提交
618
			return done(error && error.length > 0 ? error[0] : null);
E
Erich Gamma 已提交
619 620 621
		});
	}

622 623
	private matchFile(onResult: (result: IRawFileMatch) => void, candidate: IRawFileMatch): void {
		if (this.isFilePatternMatch(candidate.relativePath) && (!this.includePattern || this.includePattern(candidate.relativePath, candidate.basename))) {
624 625
			this.resultCount++;

626
			if (this.exists || (this.maxResults && this.resultCount > this.maxResults)) {
627 628 629 630
				this.isLimitHit = true;
			}

			if (!this.isLimitHit) {
631
				onResult(candidate);
632 633 634 635
			}
		}
	}

B
polish  
Benjamin Pasero 已提交
636
	private isFilePatternMatch(path: string): boolean {
637 638 639

		// Check for search pattern
		if (this.filePattern) {
640 641 642 643
			if (this.filePattern === '*') {
				return true; // support the all-matching wildcard
			}

644
			return strings.fuzzyContains(path, this.normalizedFilePatternLowercase);
645 646 647 648 649 650
		}

		// No patterns means we match all
		return true;
	}

651
	private statLinkIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error, stat: fs.Stats) => void): void {
652
		if (lstat.isSymbolicLink()) {
653
			return fs.stat(path, clb); // stat the target the link points to
654 655 656 657 658
		}

		return clb(null, lstat); // not a link, so the stat is already ok for us
	}

659
	private realPathIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error, realpath?: string) => void): void {
660 661 662 663 664
		if (lstat.isSymbolicLink()) {
			return fs.realpath(path, (error, realpath) => {
				if (error) {
					return clb(error);
				}
E
Erich Gamma 已提交
665

666 667 668
				return clb(null, realpath);
			});
		}
E
Erich Gamma 已提交
669

670
		return clb(null, path);
E
Erich Gamma 已提交
671 672 673
	}
}

674
export class Engine implements ISearchEngine<IRawFileMatch> {
675
	private folderQueries: IFolderSearch[];
676
	private extraFiles: string[];
E
Erich Gamma 已提交
677 678 679
	private walker: FileWalker;

	constructor(config: IRawSearch) {
680
		this.folderQueries = config.folderQueries;
681 682
		this.extraFiles = config.extraFiles;

E
Erich Gamma 已提交
683 684 685
		this.walker = new FileWalker(config);
	}

686
	public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchSuccess) => void): void {
687
		this.walker.walk(this.folderQueries, this.extraFiles, onResult, onProgress, (err: Error, isLimitHit: boolean) => {
C
chrmarti 已提交
688
			done(err, {
689
				type: 'success',
C
chrmarti 已提交
690 691 692 693
				limitHit: isLimitHit,
				stats: this.walker.getStats()
			});
		});
E
Erich Gamma 已提交
694 695 696 697 698
	}

	public cancel(): void {
		this.walker.cancel();
	}
699 700 701 702 703 704 705 706 707 708 709
}

/**
 * This class exists to provide one interface on top of two ParsedExpressions, one for absolute expressions and one for relative expressions.
 * The absolute and relative expressions don't "have" to be kept separate, but this keeps us from having to path.join every single
 * file searched, it's only used for a text search with a searchPath
 */
class AbsoluteAndRelativeParsedExpression {
	private absoluteParsedExpr: glob.ParsedExpression;
	private relativeParsedExpr: glob.ParsedExpression;

C
Christof Marti 已提交
710 711
	constructor(public expression: glob.IExpression, private root: string) {
		this.init(expression);
712 713 714 715 716 717 718 719
	}

	/**
	 * Split the IExpression into its absolute and relative components, and glob.parse them separately.
	 */
	private init(expr: glob.IExpression): void {
		let absoluteGlobExpr: glob.IExpression;
		let relativeGlobExpr: glob.IExpression;
720 721 722 723 724
		Object.keys(expr)
			.filter(key => expr[key])
			.forEach(key => {
				if (path.isAbsolute(key)) {
					absoluteGlobExpr = absoluteGlobExpr || glob.getEmptyExpression();
725
					absoluteGlobExpr[key] = expr[key];
726 727
				} else {
					relativeGlobExpr = relativeGlobExpr || glob.getEmptyExpression();
728
					relativeGlobExpr[key] = expr[key];
729 730
				}
			});
731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765

		this.absoluteParsedExpr = absoluteGlobExpr && glob.parse(absoluteGlobExpr, { trimForExclusions: true });
		this.relativeParsedExpr = relativeGlobExpr && glob.parse(relativeGlobExpr, { trimForExclusions: true });
	}

	public test(_path: string, basename?: string, siblingsFn?: () => string[] | TPromise<string[]>): string | TPromise<string> {
		return (this.relativeParsedExpr && this.relativeParsedExpr(_path, basename, siblingsFn)) ||
			(this.absoluteParsedExpr && this.absoluteParsedExpr(path.join(this.root, _path), basename, siblingsFn));
	}

	public getBasenameTerms(): string[] {
		const basenameTerms = [];
		if (this.absoluteParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.relativeParsedExpr));
		}

		return basenameTerms;
	}

	public getPathTerms(): string[] {
		const pathTerms = [];
		if (this.absoluteParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.relativeParsedExpr));
		}

		return pathTerms;
	}
766
}