fileSearch.ts 24.5 KB
Newer Older
E
Erich Gamma 已提交
1 2 3 4 5 6 7
/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

'use strict';

C
Christof Marti 已提交
8
import * as childProcess from 'child_process';
J
Johannes Rieken 已提交
9 10
import { StringDecoder, NodeStringDecoder } from 'string_decoder';
import { toErrorMessage } from 'vs/base/common/errorMessage';
E
Erich Gamma 已提交
11
import fs = require('fs');
12 13
import path = require('path');
import { isEqualOrParent } from 'vs/base/common/paths';
B
Benjamin Pasero 已提交
14
import { Readable } from 'stream';
15
import { TPromise } from 'vs/base/common/winjs.base';
E
Erich Gamma 已提交
16

17
import scorer = require('vs/base/common/scorer');
18
import objects = require('vs/base/common/objects');
E
Erich Gamma 已提交
19
import arrays = require('vs/base/common/arrays');
C
Christof Marti 已提交
20
import platform = require('vs/base/common/platform');
E
Erich Gamma 已提交
21
import strings = require('vs/base/common/strings');
22
import types = require('vs/base/common/types');
E
Erich Gamma 已提交
23
import glob = require('vs/base/common/glob');
J
Johannes Rieken 已提交
24
import { IProgress, IUncachedSearchStats } from 'vs/platform/search/common/search';
E
Erich Gamma 已提交
25 26 27

import extfs = require('vs/base/node/extfs');
import flow = require('vs/base/node/flow');
28
import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine, IFolderSearch } from './search';
29
import { spawnRipgrepCmd } from './ripgrepFileSearch';
E
Erich Gamma 已提交
30

C
Christof Marti 已提交
31 32 33
enum Traversal {
	Node = 1,
	MacFind,
C
Christof Marti 已提交
34
	WindowsDir,
35 36
	LinuxFind,
	Ripgrep
C
Christof Marti 已提交
37 38 39
}

interface IDirectoryEntry {
40
	base: string;
C
Christof Marti 已提交
41 42 43 44 45 46 47 48 49
	relativePath: string;
	basename: string;
}

interface IDirectoryTree {
	rootEntries: IDirectoryEntry[];
	pathToEntries: { [relativePath: string]: IDirectoryEntry[] };
}

E
Erich Gamma 已提交
50 51
export class FileWalker {
	private config: IRawSearch;
52
	private filePattern: string;
53
	private normalizedFilePatternLowercase: string;
C
Christof Marti 已提交
54
	private includePattern: glob.ParsedExpression;
E
Erich Gamma 已提交
55
	private maxResults: number;
56
	private maxFilesize: number;
E
Erich Gamma 已提交
57 58 59
	private isLimitHit: boolean;
	private resultCount: number;
	private isCanceled: boolean;
C
chrmarti 已提交
60 61 62
	private fileWalkStartTime: number;
	private directoriesWalked: number;
	private filesWalked: number;
C
Christof Marti 已提交
63 64 65 66 67
	private traversal: Traversal;
	private errors: string[];
	private cmdForkStartTime: number;
	private cmdForkResultTime: number;
	private cmdResultCount: number;
E
Erich Gamma 已提交
68

69
	private folderExcludePatterns: Map<string, AbsoluteAndRelativeParsedExpression>;
70 71
	private globalExcludePattern: glob.ParsedExpression;

E
Erich Gamma 已提交
72 73 74 75
	private walkedPaths: { [path: string]: boolean; };

	constructor(config: IRawSearch) {
		this.config = config;
76
		this.filePattern = config.filePattern;
C
Christof Marti 已提交
77
		this.includePattern = config.includePattern && glob.parse(config.includePattern);
E
Erich Gamma 已提交
78
		this.maxResults = config.maxResults || null;
79
		this.maxFilesize = config.maxFilesize || null;
E
Erich Gamma 已提交
80
		this.walkedPaths = Object.create(null);
81 82
		this.resultCount = 0;
		this.isLimitHit = false;
C
chrmarti 已提交
83 84
		this.directoriesWalked = 0;
		this.filesWalked = 0;
C
Christof Marti 已提交
85 86
		this.traversal = Traversal.Node;
		this.errors = [];
B
Benjamin Pasero 已提交
87

88 89
		if (this.filePattern) {
			this.normalizedFilePatternLowercase = strings.stripWildcards(this.filePattern).toLowerCase();
B
Benjamin Pasero 已提交
90
		}
91 92

		this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);
93
		this.folderExcludePatterns = new Map<string, AbsoluteAndRelativeParsedExpression>();
94 95

		config.folderQueries.forEach(folderQuery => {
96
			const folderExcludeExpression: glob.IExpression = objects.assign({}, folderQuery.excludePattern || {}, this.config.excludePattern || {});
97 98 99 100 101

			// Add excludes for other root folders
			config.folderQueries
				.map(rootFolderQuery => rootFolderQuery.folder)
				.filter(rootFolder => rootFolder !== folderQuery.folder)
102 103 104 105 106
				.forEach(otherRootFolder => {
					// Exclude nested root folders
					if (isEqualOrParent(otherRootFolder, folderQuery.folder)) {
						folderExcludeExpression[path.relative(folderQuery.folder, otherRootFolder)] = true;
					}
107 108
				});

109
			this.folderExcludePatterns.set(folderQuery.folder, new AbsoluteAndRelativeParsedExpression(folderExcludeExpression, folderQuery.folder));
110
		});
E
Erich Gamma 已提交
111 112 113 114 115 116
	}

	public cancel(): void {
		this.isCanceled = true;
	}

117
	public walk(folderQueries: IFolderSearch[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void {
C
chrmarti 已提交
118
		this.fileWalkStartTime = Date.now();
E
Erich Gamma 已提交
119

120
		// Support that the file pattern is a full path to a file that exists
121
		this.checkFilePatternAbsoluteMatch((exists, size) => {
122 123 124
			if (this.isCanceled) {
				return done(null, this.isLimitHit);
			}
E
Erich Gamma 已提交
125

126 127
			// Report result from file pattern if matching
			if (exists) {
128 129
				this.resultCount++;
				onResult({
130
					relativePath: this.filePattern,
131
					basename: path.basename(this.filePattern),
132 133
					size
				});
134 135 136 137

				// Optimization: a match on an absolute path is a good result and we do not
				// continue walking the entire root paths array for other matches because
				// it is very unlikely that another file would match on the full absolute path
138
				return done(null, this.isLimitHit);
139
			}
E
Erich Gamma 已提交
140

141 142 143
			// For each extra file
			if (extraFiles) {
				extraFiles.forEach(extraFilePath => {
144
					const basename = path.basename(extraFilePath);
R
Rob Lourens 已提交
145
					if (this.globalExcludePattern && this.globalExcludePattern(extraFilePath, basename)) {
146
						return; // excluded
E
Erich Gamma 已提交
147 148
					}

149
					// File: Check for match on file pattern and include pattern
150
					this.matchFile(onResult, { relativePath: extraFilePath /* no workspace relative path */, basename });
151 152
				});
			}
153

C
Christof Marti 已提交
154
			let traverse = this.nodeJSTraversal;
C
Christof Marti 已提交
155
			if (!this.maxFilesize) {
156 157 158 159
				if (this.config.useRipgrep) {
					this.traversal = Traversal.Ripgrep;
					traverse = this.cmdTraversal;
				} else if (platform.isMacintosh) {
C
Christof Marti 已提交
160
					this.traversal = Traversal.MacFind;
161
					traverse = this.cmdTraversal;
J
Johannes Rieken 已提交
162
					// Disable 'dir' for now (#11181, #11179, #11183, #11182).
163
				} /* else if (platform.isWindows) {
C
Christof Marti 已提交
164 165
					this.traversal = Traversal.WindowsDir;
					traverse = this.windowsDirTraversal;
D
Dirk Baeumer 已提交
166
				} */ else if (platform.isLinux) {
C
Christof Marti 已提交
167
					this.traversal = Traversal.LinuxFind;
168
					traverse = this.cmdTraversal;
C
Christof Marti 已提交
169 170 171 172 173 174
				}
			}

			const isNodeTraversal = traverse === this.nodeJSTraversal;
			if (!isNodeTraversal) {
				this.cmdForkStartTime = Date.now();
C
Christof Marti 已提交
175 176
			}

177
			// For each root folder
178 179
			flow.parallel<IFolderSearch, void>(folderQueries, (folderQuery: IFolderSearch, rootFolderDone: (err: Error, result: void) => void) => {
				this.call(traverse, this, folderQuery, onResult, (err?: Error) => {
C
Christof Marti 已提交
180
					if (err) {
C
Christof Marti 已提交
181
						if (isNodeTraversal) {
182
							rootFolderDone(err, undefined);
C
Christof Marti 已提交
183 184
						} else {
							// fallback
185
							const errorMessage = toErrorMessage(err);
186 187
							console.error(errorMessage);
							this.errors.push(errorMessage);
188
							this.nodeJSTraversal(folderQuery, onResult, err => rootFolderDone(err, undefined));
C
Christof Marti 已提交
189 190
						}
					} else {
191
						rootFolderDone(undefined, undefined);
192
					}
C
Christof Marti 已提交
193 194 195 196 197 198
				});
			}, (err, result) => {
				done(err ? err[0] : null, this.isLimitHit);
			});
		});
	}
199

C
Christof Marti 已提交
200 201 202 203 204 205 206 207
	private call(fun: Function, that: any, ...args: any[]): void {
		try {
			fun.apply(that, args);
		} catch (e) {
			args[args.length - 1](e);
		}
	}

208
	private cmdTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
209
		const rootFolder = folderQuery.folder;
210 211
		const isMac = platform.isMacintosh;
		let done = (err?: Error) => {
J
Johannes Rieken 已提交
212
			done = () => { };
213 214 215 216 217
			cb(err);
		};
		let leftover = '';
		let first = true;
		const tree = this.initDirectoryTree();
218

219
		const useRipgrep = this.config.useRipgrep;
220 221 222 223 224 225 226 227 228 229
		let cmd: childProcess.ChildProcess;
		let noSiblingsClauses: boolean;
		if (useRipgrep) {
			const ripgrep = spawnRipgrepCmd(folderQuery, this.config.includePattern, this.folderExcludePatterns.get(folderQuery.folder));
			cmd = ripgrep.cmd;
			noSiblingsClauses = !Object.keys(ripgrep.siblingClauses).length;
		} else {
			cmd = this.spawnFindCmd(folderQuery);
		}

230
		this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => {
C
Christof Marti 已提交
231 232 233 234
			if (err) {
				done(err);
				return;
			}
E
Erich Gamma 已提交
235

C
Christof Marti 已提交
236
			// Mac: uses NFD unicode form on disk, but we want NFC
237
			const normalized = leftover + (isMac ? strings.normalizeNFC(stdout) : stdout);
238
			const relativeFiles = normalized.split(useRipgrep ? '\n' : '\n./');
239 240 241
			if (first && normalized.length >= 2) {
				first = false;
				relativeFiles[0] = relativeFiles[0].trim().substr(2);
C
Christof Marti 已提交
242 243
			}

244 245 246 247 248 249 250 251
			if (last) {
				const n = relativeFiles.length;
				relativeFiles[n - 1] = relativeFiles[n - 1].trim();
				if (!relativeFiles[n - 1]) {
					relativeFiles.pop();
				}
			} else {
				leftover = relativeFiles.pop();
C
Christof Marti 已提交
252 253
			}

254 255 256 257 258
			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
				done(new Error('Splitting up files failed'));
				return;
			}

259 260 261 262 263 264 265 266 267 268 269 270 271
			this.cmdResultCount += relativeFiles.length;

			if (useRipgrep && noSiblingsClauses) {
				for (const relativePath of relativeFiles) {
					const basename = path.basename(relativePath);
					this.matchFile(onResult, { base: rootFolder, relativePath, basename });
				}
				if (last) {
					done();
				}
				return;
			}

272
			// TODO: Optimize siblings clauses with ripgrep here.
273
			this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
C
Christof Marti 已提交
274

275 276 277
			if (last) {
				this.matchDirectoryTree(tree, rootFolder, onResult);
				done();
278
			}
C
Christof Marti 已提交
279 280 281
		});
	}

282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
	// protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
	// 	const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
	// 	this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
	// 		if (err) {
	// 			done(err);
	// 			return;
	// 		}

	// 		const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
	// 		relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
	// 		const n = relativeFiles.length;
	// 		relativeFiles[n - 1] = relativeFiles[n - 1].trim();
	// 		if (!relativeFiles[n - 1]) {
	// 			relativeFiles.pop();
	// 		}

	// 		if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
	// 			done(new Error('Splitting up files failed'));
	// 			return;
	// 		}

	// 		this.matchFiles(rootFolder, relativeFiles, onResult);

	// 		done();
	// 	});
	// }

309 310 311
	/**
	 * Public for testing.
	 */
312
	public spawnFindCmd(folderQuery: IFolderSearch) {
313
		const excludePattern = this.folderExcludePatterns.get(folderQuery.folder);
314 315
		const basenames = excludePattern.getBasenameTerms();
		const pathTerms = excludePattern.getPathTerms();
316
		let args = ['-L', '.'];
317
		if (basenames.length || pathTerms.length) {
318
			args.push('-not', '(', '(');
319
			for (const basename of basenames) {
320
				args.push('-name', basename);
321 322
				args.push('-o');
			}
323
			for (const path of pathTerms) {
324
				args.push('-path', path);
325
				args.push('-o');
326
			}
327
			args.pop();
328 329 330
			args.push(')', '-prune', ')');
		}
		args.push('-type', 'f');
331
		return childProcess.spawn('find', args, { cwd: folderQuery.folder });
332 333 334 335 336 337
	}

	/**
	 * Public for testing.
	 */
	public readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string) => void): void {
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
		let all = '';
		this.collectStdout(cmd, encoding, (err: Error, stdout?: string, last?: boolean) => {
			if (err) {
				cb(err);
				return;
			}

			all += stdout;
			if (last) {
				cb(null, all);
			}
		});
	}

	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
		let done = (err: Error, stdout?: string, last?: boolean) => {
			if (err || last) {
J
Johannes Rieken 已提交
355
				done = () => { };
356 357 358
				this.cmdForkResultTime = Date.now();
			}
			cb(err, stdout, last);
C
Christof Marti 已提交
359 360
		};

361
		this.forwardData(cmd.stdout, encoding, done);
C
Christof Marti 已提交
362 363
		const stderr = this.collectData(cmd.stderr);

364
		cmd.on('error', (err: Error) => {
C
Christof Marti 已提交
365 366 367
			done(err);
		});

368
		cmd.on('close', (code: number) => {
C
Christof Marti 已提交
369
			if (code !== 0) {
C
Christof Marti 已提交
370
				done(new Error(`find failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
C
Christof Marti 已提交
371
			} else {
372
				done(null, '', true);
C
Christof Marti 已提交
373 374 375 376
			}
		});
	}

377 378 379 380 381 382 383 384
	private forwardData(stream: Readable, encoding: string, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
		const decoder = new StringDecoder(encoding);
		stream.on('data', (data: Buffer) => {
			cb(null, decoder.write(data));
		});
		return decoder;
	}

C
Christof Marti 已提交
385
	private collectData(stream: Readable): Buffer[] {
386 387
		const buffers: Buffer[] = [];
		stream.on('data', (data: Buffer) => {
C
Christof Marti 已提交
388 389 390 391 392
			buffers.push(data);
		});
		return buffers;
	}

C
Christof Marti 已提交
393 394 395 396 397
	private decodeData(buffers: Buffer[], encoding: string): string {
		const decoder = new StringDecoder(encoding);
		return buffers.map(buffer => decoder.write(buffer)).join('');
	}

398 399 400 401 402 403 404 405 406
	private initDirectoryTree(): IDirectoryTree {
		const tree: IDirectoryTree = {
			rootEntries: [],
			pathToEntries: Object.create(null)
		};
		tree.pathToEntries['.'] = tree.rootEntries;
		return tree;
	}

407
	private addDirectoryEntries({ pathToEntries }: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
408 409
		// Support relative paths to files from a root resource (ignores excludes)
		if (relativeFiles.indexOf(this.filePattern) !== -1) {
410
			const basename = path.basename(this.filePattern);
411
			this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
C
Christof Marti 已提交
412 413
		}

414
		function add(relativePath: string) {
415 416
			const basename = path.basename(relativePath);
			const dirname = path.dirname(relativePath);
C
Christof Marti 已提交
417 418 419 420 421 422
			let entries = pathToEntries[dirname];
			if (!entries) {
				entries = pathToEntries[dirname] = [];
				add(dirname);
			}
			entries.push({
423
				base,
C
Christof Marti 已提交
424 425 426
				relativePath,
				basename
			});
427 428
		}
		relativeFiles.forEach(add);
C
Christof Marti 已提交
429 430
	}

431
	private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
432
		const self = this;
433
		const excludePattern = this.folderExcludePatterns.get(rootFolder);
C
Christof Marti 已提交
434 435 436 437
		const filePattern = this.filePattern;
		function matchDirectory(entries: IDirectoryEntry[]) {
			self.directoriesWalked++;
			for (let i = 0, n = entries.length; i < n; i++) {
438
				const entry = entries[i];
439
				const { relativePath, basename } = entry;
C
Christof Marti 已提交
440 441 442 443 444

				// Check exclude pattern
				// If the user searches for the exact file name, we adjust the glob matching
				// to ignore filtering by siblings because the user seems to know what she
				// is searching for and we want to include the result in that case anyway
445
				if (excludePattern.test(relativePath, basename, () => filePattern !== basename ? entries.map(entry => entry.basename) : [])) {
C
Christof Marti 已提交
446 447
					continue;
				}
E
Erich Gamma 已提交
448

C
Christof Marti 已提交
449 450 451 452 453 454 455 456 457
				const sub = pathToEntries[relativePath];
				if (sub) {
					matchDirectory(sub);
				} else {
					self.filesWalked++;
					if (relativePath === filePattern) {
						continue; // ignore file if its path matches with the file pattern because that is already matched above
					}

458
					self.matchFile(onResult, entry);
C
Christof Marti 已提交
459 460 461 462 463 464
				}
			};
		}
		matchDirectory(rootEntries);
	}

465
	private nodeJSTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
C
Christof Marti 已提交
466
		this.directoriesWalked++;
467
		extfs.readdir(folderQuery.folder, (error: Error, files: string[]) => {
C
Christof Marti 已提交
468 469 470 471 472
			if (error || this.isCanceled || this.isLimitHit) {
				return done();
			}

			// Support relative paths to files from a root resource (ignores excludes)
473
			return this.checkFilePatternRelativeMatch(folderQuery.folder, (match, size) => {
C
Christof Marti 已提交
474 475 476 477 478 479 480 481
				if (this.isCanceled || this.isLimitHit) {
					return done();
				}

				// Report result from file pattern if matching
				if (match) {
					this.resultCount++;
					onResult({
482
						base: folderQuery.folder,
483
						relativePath: this.filePattern,
484
						basename: path.basename(this.filePattern),
C
Christof Marti 已提交
485
						size
486
					});
C
Christof Marti 已提交
487 488
				}

489
				return this.doWalk(folderQuery, '', files, onResult, done);
E
Erich Gamma 已提交
490
			});
491 492 493
		});
	}

494
	public getStats(): IUncachedSearchStats {
C
chrmarti 已提交
495
		return {
496
			fromCache: false,
C
Christof Marti 已提交
497 498
			traversal: Traversal[this.traversal],
			errors: this.errors,
C
chrmarti 已提交
499 500 501
			fileWalkStartTime: this.fileWalkStartTime,
			fileWalkResultTime: Date.now(),
			directoriesWalked: this.directoriesWalked,
502
			filesWalked: this.filesWalked,
C
Christof Marti 已提交
503 504 505 506
			resultCount: this.resultCount,
			cmdForkStartTime: this.cmdForkStartTime,
			cmdForkResultTime: this.cmdForkResultTime,
			cmdResultCount: this.cmdResultCount
C
chrmarti 已提交
507 508 509
		};
	}

510
	private checkFilePatternAbsoluteMatch(clb: (exists: boolean, size?: number) => void): void {
511
		if (!this.filePattern || !path.isAbsolute(this.filePattern)) {
512 513 514
			return clb(false);
		}

515
		return fs.stat(this.filePattern, (error, stat) => {
516
			return clb(!error && !stat.isDirectory(), stat && stat.size); // only existing files
E
Erich Gamma 已提交
517 518 519
		});
	}

520
	private checkFilePatternRelativeMatch(basePath: string, clb: (matchPath: string, size?: number) => void): void {
521
		if (!this.filePattern || path.isAbsolute(this.filePattern)) {
522 523 524
			return clb(null);
		}

525
		const absolutePath = path.join(basePath, this.filePattern);
526

527
		return fs.stat(absolutePath, (error, stat) => {
528
			return clb(!error && !stat.isDirectory() ? absolutePath : null, stat && stat.size); // only existing files
529 530 531
		});
	}

532 533
	private doWalk(folderQuery: IFolderSearch, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error) => void): void {
		const rootFolder = folderQuery.folder;
E
Erich Gamma 已提交
534 535

		// Execute tasks on each file in parallel to optimize throughput
536
		flow.parallel(files, (file: string, clb: (error: Error, result: {}) => void): void => {
E
Erich Gamma 已提交
537 538 539

			// Check canceled
			if (this.isCanceled || this.isLimitHit) {
540
				return clb(null, undefined);
E
Erich Gamma 已提交
541 542 543 544 545 546
			}

			// If the user searches for the exact file name, we adjust the glob matching
			// to ignore filtering by siblings because the user seems to know what she
			// is searching for and we want to include the result in that case anyway
			let siblings = files;
547
			if (this.config.filePattern === file) {
E
Erich Gamma 已提交
548 549 550 551
				siblings = [];
			}

			// Check exclude pattern
552
			let currentRelativePath = relativeParentPath ? [relativeParentPath, file].join(path.sep) : file;
553
			if (this.folderExcludePatterns.get(folderQuery.folder).test(currentRelativePath, file, () => siblings)) {
554
				return clb(null, undefined);
E
Erich Gamma 已提交
555 556
			}

557
			// Use lstat to detect links
558
			let currentAbsolutePath = [rootFolder, currentRelativePath].join(path.sep);
559
			fs.lstat(currentAbsolutePath, (error, lstat) => {
560
				if (error || this.isCanceled || this.isLimitHit) {
561
					return clb(null, undefined);
562
				}
E
Erich Gamma 已提交
563

564
				// If the path is a link, we must instead use fs.stat() to find out if the
565 566 567 568
				// link is a directory or not because lstat will always return the stat of
				// the link which is always a file.
				this.statLinkIfNeeded(currentAbsolutePath, lstat, (error, stat) => {
					if (error || this.isCanceled || this.isLimitHit) {
569
						return clb(null, undefined);
570
					}
E
Erich Gamma 已提交
571

572 573
					// Directory: Follow directories
					if (stat.isDirectory()) {
C
chrmarti 已提交
574
						this.directoriesWalked++;
575

576 577
						// to really prevent loops with links we need to resolve the real path of them
						return this.realPathIfNeeded(currentAbsolutePath, lstat, (error, realpath) => {
578
							if (error || this.isCanceled || this.isLimitHit) {
579
								return clb(null, undefined);
580 581
							}

582
							if (this.walkedPaths[realpath]) {
583
								return clb(null, undefined); // escape when there are cycles (can happen with symlinks)
584
							}
E
Erich Gamma 已提交
585

586 587 588 589 590
							this.walkedPaths[realpath] = true; // remember as walked

							// Continue walking
							return extfs.readdir(currentAbsolutePath, (error: Error, children: string[]): void => {
								if (error || this.isCanceled || this.isLimitHit) {
591
									return clb(null, undefined);
592 593
								}

594
								this.doWalk(folderQuery, currentRelativePath, children, onResult, err => clb(err, undefined));
595 596
							});
						});
597
					}
E
Erich Gamma 已提交
598

599 600
					// File: Check for match on file pattern and include pattern
					else {
C
chrmarti 已提交
601
						this.filesWalked++;
C
Christof Marti 已提交
602
						if (currentRelativePath === this.filePattern) {
603
							return clb(null, undefined); // ignore file if its path matches with the file pattern because checkFilePatternRelativeMatch() takes care of those
604
						}
E
Erich Gamma 已提交
605

606
						if (this.maxFilesize && types.isNumber(stat.size) && stat.size > this.maxFilesize) {
607
							return clb(null, undefined); // ignore file if max file size is hit
608 609
						}

610
						this.matchFile(onResult, { base: rootFolder, relativePath: currentRelativePath, basename: file, size: stat.size });
611 612 613
					}

					// Unwind
614
					return clb(null, undefined);
615
				});
E
Erich Gamma 已提交
616 617 618 619 620 621
			});
		}, (error: Error[]): void => {
			if (error) {
				error = arrays.coalesce(error); // find any error by removing null values first
			}

C
Christof Marti 已提交
622
			return done(error && error.length > 0 ? error[0] : null);
E
Erich Gamma 已提交
623 624 625
		});
	}

626 627
	private matchFile(onResult: (result: IRawFileMatch) => void, candidate: IRawFileMatch): void {
		if (this.isFilePatternMatch(candidate.relativePath) && (!this.includePattern || this.includePattern(candidate.relativePath, candidate.basename))) {
628 629 630 631 632 633 634
			this.resultCount++;

			if (this.maxResults && this.resultCount > this.maxResults) {
				this.isLimitHit = true;
			}

			if (!this.isLimitHit) {
635
				onResult(candidate);
636 637 638 639
			}
		}
	}

B
polish  
Benjamin Pasero 已提交
640
	private isFilePatternMatch(path: string): boolean {
641 642 643

		// Check for search pattern
		if (this.filePattern) {
644 645 646 647
			if (this.filePattern === '*') {
				return true; // support the all-matching wildcard
			}

648
			return scorer.matches(path, this.normalizedFilePatternLowercase);
649 650 651 652 653 654
		}

		// No patterns means we match all
		return true;
	}

655
	private statLinkIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error, stat: fs.Stats) => void): void {
656
		if (lstat.isSymbolicLink()) {
657
			return fs.stat(path, clb); // stat the target the link points to
658 659 660 661 662
		}

		return clb(null, lstat); // not a link, so the stat is already ok for us
	}

663
	private realPathIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error, realpath?: string) => void): void {
664 665 666 667 668
		if (lstat.isSymbolicLink()) {
			return fs.realpath(path, (error, realpath) => {
				if (error) {
					return clb(error);
				}
E
Erich Gamma 已提交
669

670 671 672
				return clb(null, realpath);
			});
		}
E
Erich Gamma 已提交
673

674
		return clb(null, path);
E
Erich Gamma 已提交
675 676 677
	}
}

678
export class Engine implements ISearchEngine<IRawFileMatch> {
679
	private folderQueries: IFolderSearch[];
680
	private extraFiles: string[];
E
Erich Gamma 已提交
681 682 683
	private walker: FileWalker;

	constructor(config: IRawSearch) {
684
		this.folderQueries = config.folderQueries;
685 686
		this.extraFiles = config.extraFiles;

E
Erich Gamma 已提交
687 688 689
		this.walker = new FileWalker(config);
	}

690
	public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
691
		this.walker.walk(this.folderQueries, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => {
C
chrmarti 已提交
692 693 694 695 696
			done(err, {
				limitHit: isLimitHit,
				stats: this.walker.getStats()
			});
		});
E
Erich Gamma 已提交
697 698 699 700 701
	}

	public cancel(): void {
		this.walker.cancel();
	}
702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722
}

/**
 * This class exists to provide one interface on top of two ParsedExpressions, one for absolute expressions and one for relative expressions.
 * The absolute and relative expressions don't "have" to be kept separate, but this keeps us from having to path.join every single
 * file searched, it's only used for a text search with a searchPath
 */
class AbsoluteAndRelativeParsedExpression {
	private absoluteParsedExpr: glob.ParsedExpression;
	private relativeParsedExpr: glob.ParsedExpression;

	constructor(expr: glob.IExpression, private root: string) {
		this.init(expr);
	}

	/**
	 * Split the IExpression into its absolute and relative components, and glob.parse them separately.
	 */
	private init(expr: glob.IExpression): void {
		let absoluteGlobExpr: glob.IExpression;
		let relativeGlobExpr: glob.IExpression;
723 724 725 726 727
		Object.keys(expr)
			.filter(key => expr[key])
			.forEach(key => {
				if (path.isAbsolute(key)) {
					absoluteGlobExpr = absoluteGlobExpr || glob.getEmptyExpression();
728
					absoluteGlobExpr[key] = expr[key];
729 730
				} else {
					relativeGlobExpr = relativeGlobExpr || glob.getEmptyExpression();
731
					relativeGlobExpr[key] = expr[key];
732 733
				}
			});
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768

		this.absoluteParsedExpr = absoluteGlobExpr && glob.parse(absoluteGlobExpr, { trimForExclusions: true });
		this.relativeParsedExpr = relativeGlobExpr && glob.parse(relativeGlobExpr, { trimForExclusions: true });
	}

	public test(_path: string, basename?: string, siblingsFn?: () => string[] | TPromise<string[]>): string | TPromise<string> {
		return (this.relativeParsedExpr && this.relativeParsedExpr(_path, basename, siblingsFn)) ||
			(this.absoluteParsedExpr && this.absoluteParsedExpr(path.join(this.root, _path), basename, siblingsFn));
	}

	public getBasenameTerms(): string[] {
		const basenameTerms = [];
		if (this.absoluteParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.relativeParsedExpr));
		}

		return basenameTerms;
	}

	public getPathTerms(): string[] {
		const pathTerms = [];
		if (this.absoluteParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.relativeParsedExpr));
		}

		return pathTerms;
	}
769
}