fileSearch.ts 25.0 KB
Newer Older
E
Erich Gamma 已提交
1 2 3 4 5 6 7
/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

'use strict';

C
Christof Marti 已提交
8
import * as childProcess from 'child_process';
J
Johannes Rieken 已提交
9 10
import { StringDecoder, NodeStringDecoder } from 'string_decoder';
import { toErrorMessage } from 'vs/base/common/errorMessage';
E
Erich Gamma 已提交
11
import fs = require('fs');
12 13
import path = require('path');
import { isEqualOrParent } from 'vs/base/common/paths';
B
Benjamin Pasero 已提交
14
import { Readable } from 'stream';
15
import { TPromise } from 'vs/base/common/winjs.base';
E
Erich Gamma 已提交
16

17
import scorer = require('vs/base/common/scorer');
18
import objects = require('vs/base/common/objects');
E
Erich Gamma 已提交
19
import arrays = require('vs/base/common/arrays');
C
Christof Marti 已提交
20
import platform = require('vs/base/common/platform');
E
Erich Gamma 已提交
21
import strings = require('vs/base/common/strings');
22
import types = require('vs/base/common/types');
E
Erich Gamma 已提交
23
import glob = require('vs/base/common/glob');
J
Johannes Rieken 已提交
24
import { IProgress, IUncachedSearchStats } from 'vs/platform/search/common/search';
E
Erich Gamma 已提交
25 26 27

import extfs = require('vs/base/node/extfs');
import flow = require('vs/base/node/flow');
28
import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine, IFolderSearch } from './search';
29
import { spawnRipgrepCmd } from './ripgrepFileSearch';
E
Erich Gamma 已提交
30

C
Christof Marti 已提交
31 32 33
enum Traversal {
	Node = 1,
	MacFind,
C
Christof Marti 已提交
34
	WindowsDir,
35 36
	LinuxFind,
	Ripgrep
C
Christof Marti 已提交
37 38 39
}

interface IDirectoryEntry {
40
	base: string;
C
Christof Marti 已提交
41 42 43 44 45 46 47 48 49
	relativePath: string;
	basename: string;
}

interface IDirectoryTree {
	rootEntries: IDirectoryEntry[];
	pathToEntries: { [relativePath: string]: IDirectoryEntry[] };
}

E
Erich Gamma 已提交
50 51
export class FileWalker {
	private config: IRawSearch;
C
Christof Marti 已提交
52
	private useRipgrep: boolean;
53
	private filePattern: string;
54
	private normalizedFilePatternLowercase: string;
C
Christof Marti 已提交
55
	private includePattern: glob.ParsedExpression;
E
Erich Gamma 已提交
56
	private maxResults: number;
57
	private maxFilesize: number;
E
Erich Gamma 已提交
58 59 60
	private isLimitHit: boolean;
	private resultCount: number;
	private isCanceled: boolean;
C
chrmarti 已提交
61 62 63
	private fileWalkStartTime: number;
	private directoriesWalked: number;
	private filesWalked: number;
C
Christof Marti 已提交
64 65 66 67 68
	private traversal: Traversal;
	private errors: string[];
	private cmdForkStartTime: number;
	private cmdForkResultTime: number;
	private cmdResultCount: number;
E
Erich Gamma 已提交
69

70
	private folderExcludePatterns: Map<string, AbsoluteAndRelativeParsedExpression>;
71 72
	private globalExcludePattern: glob.ParsedExpression;

E
Erich Gamma 已提交
73 74 75 76
	private walkedPaths: { [path: string]: boolean; };

	constructor(config: IRawSearch) {
		this.config = config;
C
Christof Marti 已提交
77
		this.useRipgrep = config.useRipgrep !== false;
78
		this.filePattern = config.filePattern;
C
Christof Marti 已提交
79
		this.includePattern = config.includePattern && glob.parse(config.includePattern);
E
Erich Gamma 已提交
80
		this.maxResults = config.maxResults || null;
81
		this.maxFilesize = config.maxFilesize || null;
E
Erich Gamma 已提交
82
		this.walkedPaths = Object.create(null);
83 84
		this.resultCount = 0;
		this.isLimitHit = false;
C
chrmarti 已提交
85 86
		this.directoriesWalked = 0;
		this.filesWalked = 0;
C
Christof Marti 已提交
87 88
		this.traversal = Traversal.Node;
		this.errors = [];
B
Benjamin Pasero 已提交
89

90 91
		if (this.filePattern) {
			this.normalizedFilePatternLowercase = strings.stripWildcards(this.filePattern).toLowerCase();
B
Benjamin Pasero 已提交
92
		}
93 94

		this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);
95
		this.folderExcludePatterns = new Map<string, AbsoluteAndRelativeParsedExpression>();
96 97

		config.folderQueries.forEach(folderQuery => {
98
			const folderExcludeExpression: glob.IExpression = objects.assign({}, folderQuery.excludePattern || {}, this.config.excludePattern || {});
99 100 101 102 103

			// Add excludes for other root folders
			config.folderQueries
				.map(rootFolderQuery => rootFolderQuery.folder)
				.filter(rootFolder => rootFolder !== folderQuery.folder)
104 105 106 107 108
				.forEach(otherRootFolder => {
					// Exclude nested root folders
					if (isEqualOrParent(otherRootFolder, folderQuery.folder)) {
						folderExcludeExpression[path.relative(folderQuery.folder, otherRootFolder)] = true;
					}
109 110
				});

111
			this.folderExcludePatterns.set(folderQuery.folder, new AbsoluteAndRelativeParsedExpression(folderExcludeExpression, folderQuery.folder));
112
		});
E
Erich Gamma 已提交
113 114 115 116 117 118
	}

	public cancel(): void {
		this.isCanceled = true;
	}

119
	public walk(folderQueries: IFolderSearch[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void {
C
chrmarti 已提交
120
		this.fileWalkStartTime = Date.now();
E
Erich Gamma 已提交
121

122
		// Support that the file pattern is a full path to a file that exists
123
		this.checkFilePatternAbsoluteMatch((exists, size) => {
124 125 126
			if (this.isCanceled) {
				return done(null, this.isLimitHit);
			}
E
Erich Gamma 已提交
127

128 129
			// Report result from file pattern if matching
			if (exists) {
130 131
				this.resultCount++;
				onResult({
132
					relativePath: this.filePattern,
133
					basename: path.basename(this.filePattern),
134 135
					size
				});
136 137 138 139

				// Optimization: a match on an absolute path is a good result and we do not
				// continue walking the entire root paths array for other matches because
				// it is very unlikely that another file would match on the full absolute path
140
				return done(null, this.isLimitHit);
141
			}
E
Erich Gamma 已提交
142

143 144 145
			// For each extra file
			if (extraFiles) {
				extraFiles.forEach(extraFilePath => {
146
					const basename = path.basename(extraFilePath);
R
Rob Lourens 已提交
147
					if (this.globalExcludePattern && this.globalExcludePattern(extraFilePath, basename)) {
148
						return; // excluded
E
Erich Gamma 已提交
149 150
					}

151
					// File: Check for match on file pattern and include pattern
152
					this.matchFile(onResult, { relativePath: extraFilePath /* no workspace relative path */, basename });
153 154
				});
			}
155

C
Christof Marti 已提交
156
			let traverse = this.nodeJSTraversal;
C
Christof Marti 已提交
157
			if (!this.maxFilesize) {
C
Christof Marti 已提交
158
				if (this.useRipgrep) {
159 160 161
					this.traversal = Traversal.Ripgrep;
					traverse = this.cmdTraversal;
				} else if (platform.isMacintosh) {
C
Christof Marti 已提交
162
					this.traversal = Traversal.MacFind;
163
					traverse = this.cmdTraversal;
J
Johannes Rieken 已提交
164
					// Disable 'dir' for now (#11181, #11179, #11183, #11182).
165
				} /* else if (platform.isWindows) {
C
Christof Marti 已提交
166 167
					this.traversal = Traversal.WindowsDir;
					traverse = this.windowsDirTraversal;
D
Dirk Baeumer 已提交
168
				} */ else if (platform.isLinux) {
C
Christof Marti 已提交
169
					this.traversal = Traversal.LinuxFind;
170
					traverse = this.cmdTraversal;
C
Christof Marti 已提交
171 172 173 174 175 176
				}
			}

			const isNodeTraversal = traverse === this.nodeJSTraversal;
			if (!isNodeTraversal) {
				this.cmdForkStartTime = Date.now();
C
Christof Marti 已提交
177 178
			}

179
			// For each root folder
180 181
			flow.parallel<IFolderSearch, void>(folderQueries, (folderQuery: IFolderSearch, rootFolderDone: (err: Error, result: void) => void) => {
				this.call(traverse, this, folderQuery, onResult, (err?: Error) => {
C
Christof Marti 已提交
182
					if (err) {
C
Christof Marti 已提交
183
						if (isNodeTraversal) {
184
							rootFolderDone(err, undefined);
C
Christof Marti 已提交
185 186
						} else {
							// fallback
187
							const errorMessage = toErrorMessage(err);
188 189
							console.error(errorMessage);
							this.errors.push(errorMessage);
190
							this.nodeJSTraversal(folderQuery, onResult, err => rootFolderDone(err, undefined));
C
Christof Marti 已提交
191 192
						}
					} else {
193
						rootFolderDone(undefined, undefined);
194
					}
C
Christof Marti 已提交
195 196 197 198 199 200
				});
			}, (err, result) => {
				done(err ? err[0] : null, this.isLimitHit);
			});
		});
	}
201

C
Christof Marti 已提交
202 203 204 205 206 207 208 209
	private call(fun: Function, that: any, ...args: any[]): void {
		try {
			fun.apply(that, args);
		} catch (e) {
			args[args.length - 1](e);
		}
	}

210
	private cmdTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
211
		const rootFolder = folderQuery.folder;
212 213
		const isMac = platform.isMacintosh;
		let done = (err?: Error) => {
J
Johannes Rieken 已提交
214
			done = () => { };
215 216 217 218 219
			cb(err);
		};
		let leftover = '';
		let first = true;
		const tree = this.initDirectoryTree();
220

C
Christof Marti 已提交
221
		const useRipgrep = this.useRipgrep;
222 223
		let cmd: childProcess.ChildProcess;
		let noSiblingsClauses: boolean;
C
Christof Marti 已提交
224
		let filePatternSeen = false;
225
		if (useRipgrep) {
C
Christof Marti 已提交
226
			const ripgrep = spawnRipgrepCmd(folderQuery, this.config.includePattern, this.folderExcludePatterns.get(folderQuery.folder).expression);
227 228 229 230 231 232
			cmd = ripgrep.cmd;
			noSiblingsClauses = !Object.keys(ripgrep.siblingClauses).length;
		} else {
			cmd = this.spawnFindCmd(folderQuery);
		}

233
		this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => {
C
Christof Marti 已提交
234 235 236 237
			if (err) {
				done(err);
				return;
			}
E
Erich Gamma 已提交
238

C
Christof Marti 已提交
239
			// Mac: uses NFD unicode form on disk, but we want NFC
240
			const normalized = leftover + (isMac ? strings.normalizeNFC(stdout) : stdout);
241
			const relativeFiles = normalized.split(useRipgrep ? '\n' : '\n./');
C
Christof Marti 已提交
242
			if (!useRipgrep && first && normalized.length >= 2) {
243 244
				first = false;
				relativeFiles[0] = relativeFiles[0].trim().substr(2);
C
Christof Marti 已提交
245 246
			}

247 248 249 250 251 252 253 254
			if (last) {
				const n = relativeFiles.length;
				relativeFiles[n - 1] = relativeFiles[n - 1].trim();
				if (!relativeFiles[n - 1]) {
					relativeFiles.pop();
				}
			} else {
				leftover = relativeFiles.pop();
C
Christof Marti 已提交
255 256
			}

257 258 259 260 261
			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
				done(new Error('Splitting up files failed'));
				return;
			}

262 263 264 265
			this.cmdResultCount += relativeFiles.length;

			if (useRipgrep && noSiblingsClauses) {
				for (const relativePath of relativeFiles) {
C
Christof Marti 已提交
266 267 268
					if (relativePath === this.filePattern) {
						filePatternSeen = true;
					}
269 270 271 272
					const basename = path.basename(relativePath);
					this.matchFile(onResult, { base: rootFolder, relativePath, basename });
				}
				if (last) {
C
Christof Marti 已提交
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
					if (!filePatternSeen) {
						this.checkFilePatternRelativeMatch(folderQuery.folder, (match, size) => {
							if (match) {
								this.resultCount++;
								onResult({
									base: folderQuery.folder,
									relativePath: this.filePattern,
									basename: path.basename(this.filePattern),
								});
							}
							done();
						});
					} else {
						done();
					}
288 289 290 291
				}
				return;
			}

292
			// TODO: Optimize siblings clauses with ripgrep here.
293
			this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
C
Christof Marti 已提交
294

295 296 297
			if (last) {
				this.matchDirectoryTree(tree, rootFolder, onResult);
				done();
298
			}
C
Christof Marti 已提交
299 300 301
		});
	}

302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
	// protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
	// 	const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
	// 	this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
	// 		if (err) {
	// 			done(err);
	// 			return;
	// 		}

	// 		const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
	// 		relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
	// 		const n = relativeFiles.length;
	// 		relativeFiles[n - 1] = relativeFiles[n - 1].trim();
	// 		if (!relativeFiles[n - 1]) {
	// 			relativeFiles.pop();
	// 		}

	// 		if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
	// 			done(new Error('Splitting up files failed'));
	// 			return;
	// 		}

	// 		this.matchFiles(rootFolder, relativeFiles, onResult);

	// 		done();
	// 	});
	// }

329 330 331
	/**
	 * Public for testing.
	 */
332
	public spawnFindCmd(folderQuery: IFolderSearch) {
333
		const excludePattern = this.folderExcludePatterns.get(folderQuery.folder);
334 335
		const basenames = excludePattern.getBasenameTerms();
		const pathTerms = excludePattern.getPathTerms();
336
		let args = ['-L', '.'];
337
		if (basenames.length || pathTerms.length) {
338
			args.push('-not', '(', '(');
339
			for (const basename of basenames) {
340
				args.push('-name', basename);
341 342
				args.push('-o');
			}
343
			for (const path of pathTerms) {
344
				args.push('-path', path);
345
				args.push('-o');
346
			}
347
			args.pop();
348 349 350
			args.push(')', '-prune', ')');
		}
		args.push('-type', 'f');
351
		return childProcess.spawn('find', args, { cwd: folderQuery.folder });
352 353 354 355 356 357
	}

	/**
	 * Public for testing.
	 */
	public readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string) => void): void {
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374
		let all = '';
		this.collectStdout(cmd, encoding, (err: Error, stdout?: string, last?: boolean) => {
			if (err) {
				cb(err);
				return;
			}

			all += stdout;
			if (last) {
				cb(null, all);
			}
		});
	}

	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
		let done = (err: Error, stdout?: string, last?: boolean) => {
			if (err || last) {
J
Johannes Rieken 已提交
375
				done = () => { };
376 377 378
				this.cmdForkResultTime = Date.now();
			}
			cb(err, stdout, last);
C
Christof Marti 已提交
379 380
		};

381
		this.forwardData(cmd.stdout, encoding, done);
C
Christof Marti 已提交
382 383
		const stderr = this.collectData(cmd.stderr);

384
		cmd.on('error', (err: Error) => {
C
Christof Marti 已提交
385 386 387
			done(err);
		});

388
		cmd.on('close', (code: number) => {
C
Christof Marti 已提交
389
			if (code !== 0) {
C
Christof Marti 已提交
390
				done(new Error(`command failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
C
Christof Marti 已提交
391
			} else {
392
				done(null, '', true);
C
Christof Marti 已提交
393 394 395 396
			}
		});
	}

397 398 399 400 401 402 403 404
	private forwardData(stream: Readable, encoding: string, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
		const decoder = new StringDecoder(encoding);
		stream.on('data', (data: Buffer) => {
			cb(null, decoder.write(data));
		});
		return decoder;
	}

C
Christof Marti 已提交
405
	private collectData(stream: Readable): Buffer[] {
406 407
		const buffers: Buffer[] = [];
		stream.on('data', (data: Buffer) => {
C
Christof Marti 已提交
408 409 410 411 412
			buffers.push(data);
		});
		return buffers;
	}

C
Christof Marti 已提交
413 414 415 416 417
	private decodeData(buffers: Buffer[], encoding: string): string {
		const decoder = new StringDecoder(encoding);
		return buffers.map(buffer => decoder.write(buffer)).join('');
	}

418 419 420 421 422 423 424 425 426
	private initDirectoryTree(): IDirectoryTree {
		const tree: IDirectoryTree = {
			rootEntries: [],
			pathToEntries: Object.create(null)
		};
		tree.pathToEntries['.'] = tree.rootEntries;
		return tree;
	}

427
	private addDirectoryEntries({ pathToEntries }: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
428 429
		// Support relative paths to files from a root resource (ignores excludes)
		if (relativeFiles.indexOf(this.filePattern) !== -1) {
430
			const basename = path.basename(this.filePattern);
431
			this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
C
Christof Marti 已提交
432 433
		}

434
		function add(relativePath: string) {
435 436
			const basename = path.basename(relativePath);
			const dirname = path.dirname(relativePath);
C
Christof Marti 已提交
437 438 439 440 441 442
			let entries = pathToEntries[dirname];
			if (!entries) {
				entries = pathToEntries[dirname] = [];
				add(dirname);
			}
			entries.push({
443
				base,
C
Christof Marti 已提交
444 445 446
				relativePath,
				basename
			});
447 448
		}
		relativeFiles.forEach(add);
C
Christof Marti 已提交
449 450
	}

451
	private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
452
		const self = this;
453
		const excludePattern = this.folderExcludePatterns.get(rootFolder);
C
Christof Marti 已提交
454 455 456 457
		const filePattern = this.filePattern;
		function matchDirectory(entries: IDirectoryEntry[]) {
			self.directoriesWalked++;
			for (let i = 0, n = entries.length; i < n; i++) {
458
				const entry = entries[i];
459
				const { relativePath, basename } = entry;
C
Christof Marti 已提交
460 461 462 463 464

				// Check exclude pattern
				// If the user searches for the exact file name, we adjust the glob matching
				// to ignore filtering by siblings because the user seems to know what she
				// is searching for and we want to include the result in that case anyway
465
				if (excludePattern.test(relativePath, basename, () => filePattern !== basename ? entries.map(entry => entry.basename) : [])) {
C
Christof Marti 已提交
466 467
					continue;
				}
E
Erich Gamma 已提交
468

C
Christof Marti 已提交
469 470 471 472 473 474 475 476 477
				const sub = pathToEntries[relativePath];
				if (sub) {
					matchDirectory(sub);
				} else {
					self.filesWalked++;
					if (relativePath === filePattern) {
						continue; // ignore file if its path matches with the file pattern because that is already matched above
					}

478
					self.matchFile(onResult, entry);
C
Christof Marti 已提交
479 480 481 482 483 484
				}
			};
		}
		matchDirectory(rootEntries);
	}

485
	private nodeJSTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
C
Christof Marti 已提交
486
		this.directoriesWalked++;
487
		extfs.readdir(folderQuery.folder, (error: Error, files: string[]) => {
C
Christof Marti 已提交
488 489 490 491 492
			if (error || this.isCanceled || this.isLimitHit) {
				return done();
			}

			// Support relative paths to files from a root resource (ignores excludes)
493
			return this.checkFilePatternRelativeMatch(folderQuery.folder, (match, size) => {
C
Christof Marti 已提交
494 495 496 497 498 499 500 501
				if (this.isCanceled || this.isLimitHit) {
					return done();
				}

				// Report result from file pattern if matching
				if (match) {
					this.resultCount++;
					onResult({
502
						base: folderQuery.folder,
503
						relativePath: this.filePattern,
504
						basename: path.basename(this.filePattern),
C
Christof Marti 已提交
505
						size
506
					});
C
Christof Marti 已提交
507 508
				}

509
				return this.doWalk(folderQuery, '', files, onResult, done);
E
Erich Gamma 已提交
510
			});
511 512 513
		});
	}

514
	public getStats(): IUncachedSearchStats {
C
chrmarti 已提交
515
		return {
516
			fromCache: false,
C
Christof Marti 已提交
517 518
			traversal: Traversal[this.traversal],
			errors: this.errors,
C
chrmarti 已提交
519 520 521
			fileWalkStartTime: this.fileWalkStartTime,
			fileWalkResultTime: Date.now(),
			directoriesWalked: this.directoriesWalked,
522
			filesWalked: this.filesWalked,
C
Christof Marti 已提交
523 524 525 526
			resultCount: this.resultCount,
			cmdForkStartTime: this.cmdForkStartTime,
			cmdForkResultTime: this.cmdForkResultTime,
			cmdResultCount: this.cmdResultCount
C
chrmarti 已提交
527 528 529
		};
	}

530
	private checkFilePatternAbsoluteMatch(clb: (exists: boolean, size?: number) => void): void {
531
		if (!this.filePattern || !path.isAbsolute(this.filePattern)) {
532 533 534
			return clb(false);
		}

535
		return fs.stat(this.filePattern, (error, stat) => {
536
			return clb(!error && !stat.isDirectory(), stat && stat.size); // only existing files
E
Erich Gamma 已提交
537 538 539
		});
	}

540
	private checkFilePatternRelativeMatch(basePath: string, clb: (matchPath: string, size?: number) => void): void {
541
		if (!this.filePattern || path.isAbsolute(this.filePattern)) {
542 543 544
			return clb(null);
		}

545
		const absolutePath = path.join(basePath, this.filePattern);
546

547
		return fs.stat(absolutePath, (error, stat) => {
548
			return clb(!error && !stat.isDirectory() ? absolutePath : null, stat && stat.size); // only existing files
549 550 551
		});
	}

552 553
	private doWalk(folderQuery: IFolderSearch, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error) => void): void {
		const rootFolder = folderQuery.folder;
E
Erich Gamma 已提交
554 555

		// Execute tasks on each file in parallel to optimize throughput
556
		flow.parallel(files, (file: string, clb: (error: Error, result: {}) => void): void => {
E
Erich Gamma 已提交
557 558 559

			// Check canceled
			if (this.isCanceled || this.isLimitHit) {
560
				return clb(null, undefined);
E
Erich Gamma 已提交
561 562 563 564 565 566
			}

			// If the user searches for the exact file name, we adjust the glob matching
			// to ignore filtering by siblings because the user seems to know what she
			// is searching for and we want to include the result in that case anyway
			let siblings = files;
567
			if (this.config.filePattern === file) {
E
Erich Gamma 已提交
568 569 570 571
				siblings = [];
			}

			// Check exclude pattern
572
			let currentRelativePath = relativeParentPath ? [relativeParentPath, file].join(path.sep) : file;
573
			if (this.folderExcludePatterns.get(folderQuery.folder).test(currentRelativePath, file, () => siblings)) {
574
				return clb(null, undefined);
E
Erich Gamma 已提交
575 576
			}

577
			// Use lstat to detect links
578
			let currentAbsolutePath = [rootFolder, currentRelativePath].join(path.sep);
579
			fs.lstat(currentAbsolutePath, (error, lstat) => {
580
				if (error || this.isCanceled || this.isLimitHit) {
581
					return clb(null, undefined);
582
				}
E
Erich Gamma 已提交
583

584
				// If the path is a link, we must instead use fs.stat() to find out if the
585 586 587 588
				// link is a directory or not because lstat will always return the stat of
				// the link which is always a file.
				this.statLinkIfNeeded(currentAbsolutePath, lstat, (error, stat) => {
					if (error || this.isCanceled || this.isLimitHit) {
589
						return clb(null, undefined);
590
					}
E
Erich Gamma 已提交
591

592 593
					// Directory: Follow directories
					if (stat.isDirectory()) {
C
chrmarti 已提交
594
						this.directoriesWalked++;
595

596 597
						// to really prevent loops with links we need to resolve the real path of them
						return this.realPathIfNeeded(currentAbsolutePath, lstat, (error, realpath) => {
598
							if (error || this.isCanceled || this.isLimitHit) {
599
								return clb(null, undefined);
600 601
							}

602
							if (this.walkedPaths[realpath]) {
603
								return clb(null, undefined); // escape when there are cycles (can happen with symlinks)
604
							}
E
Erich Gamma 已提交
605

606 607 608 609 610
							this.walkedPaths[realpath] = true; // remember as walked

							// Continue walking
							return extfs.readdir(currentAbsolutePath, (error: Error, children: string[]): void => {
								if (error || this.isCanceled || this.isLimitHit) {
611
									return clb(null, undefined);
612 613
								}

614
								this.doWalk(folderQuery, currentRelativePath, children, onResult, err => clb(err, undefined));
615 616
							});
						});
617
					}
E
Erich Gamma 已提交
618

619 620
					// File: Check for match on file pattern and include pattern
					else {
C
chrmarti 已提交
621
						this.filesWalked++;
C
Christof Marti 已提交
622
						if (currentRelativePath === this.filePattern) {
623
							return clb(null, undefined); // ignore file if its path matches with the file pattern because checkFilePatternRelativeMatch() takes care of those
624
						}
E
Erich Gamma 已提交
625

626
						if (this.maxFilesize && types.isNumber(stat.size) && stat.size > this.maxFilesize) {
627
							return clb(null, undefined); // ignore file if max file size is hit
628 629
						}

630
						this.matchFile(onResult, { base: rootFolder, relativePath: currentRelativePath, basename: file, size: stat.size });
631 632 633
					}

					// Unwind
634
					return clb(null, undefined);
635
				});
E
Erich Gamma 已提交
636 637 638 639 640 641
			});
		}, (error: Error[]): void => {
			if (error) {
				error = arrays.coalesce(error); // find any error by removing null values first
			}

C
Christof Marti 已提交
642
			return done(error && error.length > 0 ? error[0] : null);
E
Erich Gamma 已提交
643 644 645
		});
	}

646 647
	private matchFile(onResult: (result: IRawFileMatch) => void, candidate: IRawFileMatch): void {
		if (this.isFilePatternMatch(candidate.relativePath) && (!this.includePattern || this.includePattern(candidate.relativePath, candidate.basename))) {
648 649 650 651 652 653 654
			this.resultCount++;

			if (this.maxResults && this.resultCount > this.maxResults) {
				this.isLimitHit = true;
			}

			if (!this.isLimitHit) {
655
				onResult(candidate);
656 657 658 659
			}
		}
	}

B
polish  
Benjamin Pasero 已提交
660
	private isFilePatternMatch(path: string): boolean {
661 662 663

		// Check for search pattern
		if (this.filePattern) {
664 665 666 667
			if (this.filePattern === '*') {
				return true; // support the all-matching wildcard
			}

668
			return scorer.matches(path, this.normalizedFilePatternLowercase);
669 670 671 672 673 674
		}

		// No patterns means we match all
		return true;
	}

675
	private statLinkIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error, stat: fs.Stats) => void): void {
676
		if (lstat.isSymbolicLink()) {
677
			return fs.stat(path, clb); // stat the target the link points to
678 679 680 681 682
		}

		return clb(null, lstat); // not a link, so the stat is already ok for us
	}

683
	private realPathIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error, realpath?: string) => void): void {
684 685 686 687 688
		if (lstat.isSymbolicLink()) {
			return fs.realpath(path, (error, realpath) => {
				if (error) {
					return clb(error);
				}
E
Erich Gamma 已提交
689

690 691 692
				return clb(null, realpath);
			});
		}
E
Erich Gamma 已提交
693

694
		return clb(null, path);
E
Erich Gamma 已提交
695 696 697
	}
}

698
export class Engine implements ISearchEngine<IRawFileMatch> {
699
	private folderQueries: IFolderSearch[];
700
	private extraFiles: string[];
E
Erich Gamma 已提交
701 702 703
	private walker: FileWalker;

	constructor(config: IRawSearch) {
704
		this.folderQueries = config.folderQueries;
705 706
		this.extraFiles = config.extraFiles;

E
Erich Gamma 已提交
707 708 709
		this.walker = new FileWalker(config);
	}

710
	public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
711
		this.walker.walk(this.folderQueries, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => {
C
chrmarti 已提交
712 713 714 715 716
			done(err, {
				limitHit: isLimitHit,
				stats: this.walker.getStats()
			});
		});
E
Erich Gamma 已提交
717 718 719 720 721
	}

	public cancel(): void {
		this.walker.cancel();
	}
722 723 724 725 726 727 728 729 730 731 732
}

/**
 * This class exists to provide one interface on top of two ParsedExpressions, one for absolute expressions and one for relative expressions.
 * The absolute and relative expressions don't "have" to be kept separate, but this keeps us from having to path.join every single
 * file searched, it's only used for a text search with a searchPath
 */
class AbsoluteAndRelativeParsedExpression {
	private absoluteParsedExpr: glob.ParsedExpression;
	private relativeParsedExpr: glob.ParsedExpression;

C
Christof Marti 已提交
733 734
	constructor(public expression: glob.IExpression, private root: string) {
		this.init(expression);
735 736 737 738 739 740 741 742
	}

	/**
	 * Split the IExpression into its absolute and relative components, and glob.parse them separately.
	 */
	private init(expr: glob.IExpression): void {
		let absoluteGlobExpr: glob.IExpression;
		let relativeGlobExpr: glob.IExpression;
743 744 745 746 747
		Object.keys(expr)
			.filter(key => expr[key])
			.forEach(key => {
				if (path.isAbsolute(key)) {
					absoluteGlobExpr = absoluteGlobExpr || glob.getEmptyExpression();
748
					absoluteGlobExpr[key] = expr[key];
749 750
				} else {
					relativeGlobExpr = relativeGlobExpr || glob.getEmptyExpression();
751
					relativeGlobExpr[key] = expr[key];
752 753
				}
			});
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788

		this.absoluteParsedExpr = absoluteGlobExpr && glob.parse(absoluteGlobExpr, { trimForExclusions: true });
		this.relativeParsedExpr = relativeGlobExpr && glob.parse(relativeGlobExpr, { trimForExclusions: true });
	}

	public test(_path: string, basename?: string, siblingsFn?: () => string[] | TPromise<string[]>): string | TPromise<string> {
		return (this.relativeParsedExpr && this.relativeParsedExpr(_path, basename, siblingsFn)) ||
			(this.absoluteParsedExpr && this.absoluteParsedExpr(path.join(this.root, _path), basename, siblingsFn));
	}

	public getBasenameTerms(): string[] {
		const basenameTerms = [];
		if (this.absoluteParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.relativeParsedExpr));
		}

		return basenameTerms;
	}

	public getPathTerms(): string[] {
		const pathTerms = [];
		if (this.absoluteParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.relativeParsedExpr));
		}

		return pathTerms;
	}
789
}