fileSearch.ts 24.0 KB
Newer Older
E
Erich Gamma 已提交
1 2 3 4 5 6 7
/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

'use strict';

C
Christof Marti 已提交
8
import * as childProcess from 'child_process';
J
Johannes Rieken 已提交
9 10
import { StringDecoder, NodeStringDecoder } from 'string_decoder';
import { toErrorMessage } from 'vs/base/common/errorMessage';
E
Erich Gamma 已提交
11
import fs = require('fs');
12 13
import path = require('path');
import { isEqualOrParent } from 'vs/base/common/paths';
B
Benjamin Pasero 已提交
14
import { Readable } from 'stream';
15
import { TPromise } from 'vs/base/common/winjs.base';
E
Erich Gamma 已提交
16

17
import scorer = require('vs/base/common/scorer');
18
import objects = require('vs/base/common/objects');
E
Erich Gamma 已提交
19
import arrays = require('vs/base/common/arrays');
C
Christof Marti 已提交
20
import platform = require('vs/base/common/platform');
E
Erich Gamma 已提交
21
import strings = require('vs/base/common/strings');
22
import types = require('vs/base/common/types');
E
Erich Gamma 已提交
23
import glob = require('vs/base/common/glob');
J
Johannes Rieken 已提交
24
import { IProgress, IUncachedSearchStats } from 'vs/platform/search/common/search';
E
Erich Gamma 已提交
25 26 27

import extfs = require('vs/base/node/extfs');
import flow = require('vs/base/node/flow');
28
import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine, IFolderSearch } from './search';
29
import { spawnRipgrepCmd } from './ripgrepFileSearch';
E
Erich Gamma 已提交
30

C
Christof Marti 已提交
31 32 33
enum Traversal {
	Node = 1,
	MacFind,
C
Christof Marti 已提交
34
	WindowsDir,
35 36
	LinuxFind,
	Ripgrep
C
Christof Marti 已提交
37 38 39
}

interface IDirectoryEntry {
40
	base: string;
C
Christof Marti 已提交
41 42 43 44 45 46 47 48 49
	relativePath: string;
	basename: string;
}

interface IDirectoryTree {
	rootEntries: IDirectoryEntry[];
	pathToEntries: { [relativePath: string]: IDirectoryEntry[] };
}

E
Erich Gamma 已提交
50 51
export class FileWalker {
	private config: IRawSearch;
52
	private filePattern: string;
53
	private normalizedFilePatternLowercase: string;
C
Christof Marti 已提交
54
	private includePattern: glob.ParsedExpression;
E
Erich Gamma 已提交
55
	private maxResults: number;
56
	private maxFilesize: number;
E
Erich Gamma 已提交
57 58 59
	private isLimitHit: boolean;
	private resultCount: number;
	private isCanceled: boolean;
C
chrmarti 已提交
60 61 62
	private fileWalkStartTime: number;
	private directoriesWalked: number;
	private filesWalked: number;
C
Christof Marti 已提交
63 64 65 66 67
	private traversal: Traversal;
	private errors: string[];
	private cmdForkStartTime: number;
	private cmdForkResultTime: number;
	private cmdResultCount: number;
E
Erich Gamma 已提交
68

69
	private folderExcludePatterns: Map<string, AbsoluteAndRelativeParsedExpression>;
70 71
	private globalExcludePattern: glob.ParsedExpression;

E
Erich Gamma 已提交
72 73 74 75
	private walkedPaths: { [path: string]: boolean; };

	constructor(config: IRawSearch) {
		this.config = config;
76
		this.filePattern = config.filePattern;
C
Christof Marti 已提交
77
		this.includePattern = config.includePattern && glob.parse(config.includePattern);
E
Erich Gamma 已提交
78
		this.maxResults = config.maxResults || null;
79
		this.maxFilesize = config.maxFilesize || null;
E
Erich Gamma 已提交
80
		this.walkedPaths = Object.create(null);
81 82
		this.resultCount = 0;
		this.isLimitHit = false;
C
chrmarti 已提交
83 84
		this.directoriesWalked = 0;
		this.filesWalked = 0;
C
Christof Marti 已提交
85 86
		this.traversal = Traversal.Node;
		this.errors = [];
B
Benjamin Pasero 已提交
87

88 89
		if (this.filePattern) {
			this.normalizedFilePatternLowercase = strings.stripWildcards(this.filePattern).toLowerCase();
B
Benjamin Pasero 已提交
90
		}
91 92

		this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);
93
		this.folderExcludePatterns = new Map<string, AbsoluteAndRelativeParsedExpression>();
94 95

		config.folderQueries.forEach(folderQuery => {
96
			const folderExcludeExpression: glob.IExpression = objects.assign({}, folderQuery.excludePattern || {}, this.config.excludePattern || {});
97 98 99 100 101

			// Add excludes for other root folders
			config.folderQueries
				.map(rootFolderQuery => rootFolderQuery.folder)
				.filter(rootFolder => rootFolder !== folderQuery.folder)
102 103 104 105 106
				.forEach(otherRootFolder => {
					// Exclude nested root folders
					if (isEqualOrParent(otherRootFolder, folderQuery.folder)) {
						folderExcludeExpression[path.relative(folderQuery.folder, otherRootFolder)] = true;
					}
107 108
				});

109
			this.folderExcludePatterns.set(folderQuery.folder, new AbsoluteAndRelativeParsedExpression(folderExcludeExpression, folderQuery.folder));
110
		});
E
Erich Gamma 已提交
111 112 113 114 115 116
	}

	public cancel(): void {
		this.isCanceled = true;
	}

117
	public walk(folderQueries: IFolderSearch[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void {
C
chrmarti 已提交
118
		this.fileWalkStartTime = Date.now();
E
Erich Gamma 已提交
119

120
		// Support that the file pattern is a full path to a file that exists
121
		this.checkFilePatternAbsoluteMatch((exists, size) => {
122 123 124
			if (this.isCanceled) {
				return done(null, this.isLimitHit);
			}
E
Erich Gamma 已提交
125

126 127
			// Report result from file pattern if matching
			if (exists) {
128 129
				this.resultCount++;
				onResult({
130
					relativePath: this.filePattern,
131
					basename: path.basename(this.filePattern),
132 133
					size
				});
134 135 136 137

				// Optimization: a match on an absolute path is a good result and we do not
				// continue walking the entire root paths array for other matches because
				// it is very unlikely that another file would match on the full absolute path
138
				return done(null, this.isLimitHit);
139
			}
E
Erich Gamma 已提交
140

141 142 143
			// For each extra file
			if (extraFiles) {
				extraFiles.forEach(extraFilePath => {
144
					const basename = path.basename(extraFilePath);
R
Rob Lourens 已提交
145
					if (this.globalExcludePattern && this.globalExcludePattern(extraFilePath, basename)) {
146
						return; // excluded
E
Erich Gamma 已提交
147 148
					}

149
					// File: Check for match on file pattern and include pattern
150
					this.matchFile(onResult, { relativePath: extraFilePath /* no workspace relative path */, basename });
151 152
				});
			}
153

C
Christof Marti 已提交
154
			let traverse = this.nodeJSTraversal;
C
Christof Marti 已提交
155
			if (!this.maxFilesize) {
156 157 158 159
				if (this.config.useRipgrep) {
					this.traversal = Traversal.Ripgrep;
					traverse = this.cmdTraversal;
				} else if (platform.isMacintosh) {
C
Christof Marti 已提交
160
					this.traversal = Traversal.MacFind;
161
					traverse = this.cmdTraversal;
J
Johannes Rieken 已提交
162
					// Disable 'dir' for now (#11181, #11179, #11183, #11182).
163
				} /* else if (platform.isWindows) {
C
Christof Marti 已提交
164 165
					this.traversal = Traversal.WindowsDir;
					traverse = this.windowsDirTraversal;
D
Dirk Baeumer 已提交
166
				} */ else if (platform.isLinux) {
C
Christof Marti 已提交
167
					this.traversal = Traversal.LinuxFind;
168
					traverse = this.cmdTraversal;
C
Christof Marti 已提交
169 170 171 172 173 174
				}
			}

			const isNodeTraversal = traverse === this.nodeJSTraversal;
			if (!isNodeTraversal) {
				this.cmdForkStartTime = Date.now();
C
Christof Marti 已提交
175 176
			}

177
			// For each root folder
178 179
			flow.parallel<IFolderSearch, void>(folderQueries, (folderQuery: IFolderSearch, rootFolderDone: (err: Error, result: void) => void) => {
				this.call(traverse, this, folderQuery, onResult, (err?: Error) => {
C
Christof Marti 已提交
180
					if (err) {
C
Christof Marti 已提交
181
						if (isNodeTraversal) {
182
							rootFolderDone(err, undefined);
C
Christof Marti 已提交
183 184
						} else {
							// fallback
185
							const errorMessage = toErrorMessage(err);
186 187
							console.error(errorMessage);
							this.errors.push(errorMessage);
188
							this.nodeJSTraversal(folderQuery, onResult, err => rootFolderDone(err, undefined));
C
Christof Marti 已提交
189 190
						}
					} else {
191
						rootFolderDone(undefined, undefined);
192
					}
C
Christof Marti 已提交
193 194 195 196 197 198
				});
			}, (err, result) => {
				done(err ? err[0] : null, this.isLimitHit);
			});
		});
	}
199

C
Christof Marti 已提交
200 201 202 203 204 205 206 207
	private call(fun: Function, that: any, ...args: any[]): void {
		try {
			fun.apply(that, args);
		} catch (e) {
			args[args.length - 1](e);
		}
	}

208
	private cmdTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
209
		const rootFolder = folderQuery.folder;
210 211
		const isMac = platform.isMacintosh;
		let done = (err?: Error) => {
J
Johannes Rieken 已提交
212
			done = () => { };
213 214 215 216 217
			cb(err);
		};
		let leftover = '';
		let first = true;
		const tree = this.initDirectoryTree();
218 219
		const useRipgrep = this.config.useRipgrep;
		const cmd = useRipgrep ? spawnRipgrepCmd(folderQuery, this.config.includePattern, this.folderExcludePatterns.get(folderQuery.folder)).cmd : this.spawnFindCmd(folderQuery);
220
		this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => {
C
Christof Marti 已提交
221 222 223 224
			if (err) {
				done(err);
				return;
			}
E
Erich Gamma 已提交
225

C
Christof Marti 已提交
226
			// Mac: uses NFD unicode form on disk, but we want NFC
227
			const normalized = leftover + (isMac ? strings.normalizeNFC(stdout) : stdout);
228
			const relativeFiles = normalized.split(useRipgrep ? '\n' : '\n./');
229 230 231
			if (first && normalized.length >= 2) {
				first = false;
				relativeFiles[0] = relativeFiles[0].trim().substr(2);
C
Christof Marti 已提交
232 233
			}

234 235 236 237 238 239 240 241
			if (last) {
				const n = relativeFiles.length;
				relativeFiles[n - 1] = relativeFiles[n - 1].trim();
				if (!relativeFiles[n - 1]) {
					relativeFiles.pop();
				}
			} else {
				leftover = relativeFiles.pop();
C
Christof Marti 已提交
242 243
			}

244 245 246 247 248
			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
				done(new Error('Splitting up files failed'));
				return;
			}

249
			// TODO: Optimize siblings clauses with ripgrep here.
250
			this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
C
Christof Marti 已提交
251

252 253 254
			if (last) {
				this.matchDirectoryTree(tree, rootFolder, onResult);
				done();
255
			}
C
Christof Marti 已提交
256 257 258
		});
	}

259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
	// protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
	// 	const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
	// 	this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
	// 		if (err) {
	// 			done(err);
	// 			return;
	// 		}

	// 		const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
	// 		relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
	// 		const n = relativeFiles.length;
	// 		relativeFiles[n - 1] = relativeFiles[n - 1].trim();
	// 		if (!relativeFiles[n - 1]) {
	// 			relativeFiles.pop();
	// 		}

	// 		if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
	// 			done(new Error('Splitting up files failed'));
	// 			return;
	// 		}

	// 		this.matchFiles(rootFolder, relativeFiles, onResult);

	// 		done();
	// 	});
	// }

286 287 288
	/**
	 * Public for testing.
	 */
289
	public spawnFindCmd(folderQuery: IFolderSearch) {
290
		const excludePattern = this.folderExcludePatterns.get(folderQuery.folder);
291 292
		const basenames = excludePattern.getBasenameTerms();
		const pathTerms = excludePattern.getPathTerms();
293
		let args = ['-L', '.'];
294
		if (basenames.length || pathTerms.length) {
295
			args.push('-not', '(', '(');
296
			for (const basename of basenames) {
297
				args.push('-name', basename);
298 299
				args.push('-o');
			}
300
			for (const path of pathTerms) {
301
				args.push('-path', path);
302
				args.push('-o');
303
			}
304
			args.pop();
305 306 307
			args.push(')', '-prune', ')');
		}
		args.push('-type', 'f');
308
		return childProcess.spawn('find', args, { cwd: folderQuery.folder });
309 310 311 312 313 314
	}

	/**
	 * Public for testing.
	 */
	public readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string) => void): void {
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
		let all = '';
		this.collectStdout(cmd, encoding, (err: Error, stdout?: string, last?: boolean) => {
			if (err) {
				cb(err);
				return;
			}

			all += stdout;
			if (last) {
				cb(null, all);
			}
		});
	}

	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
		let done = (err: Error, stdout?: string, last?: boolean) => {
			if (err || last) {
J
Johannes Rieken 已提交
332
				done = () => { };
333 334 335
				this.cmdForkResultTime = Date.now();
			}
			cb(err, stdout, last);
C
Christof Marti 已提交
336 337
		};

338
		this.forwardData(cmd.stdout, encoding, done);
C
Christof Marti 已提交
339 340
		const stderr = this.collectData(cmd.stderr);

341
		cmd.on('error', (err: Error) => {
C
Christof Marti 已提交
342 343 344
			done(err);
		});

345
		cmd.on('close', (code: number) => {
C
Christof Marti 已提交
346
			if (code !== 0) {
C
Christof Marti 已提交
347
				done(new Error(`find failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
C
Christof Marti 已提交
348
			} else {
349
				done(null, '', true);
C
Christof Marti 已提交
350 351 352 353
			}
		});
	}

354 355 356 357 358 359 360 361
	private forwardData(stream: Readable, encoding: string, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
		const decoder = new StringDecoder(encoding);
		stream.on('data', (data: Buffer) => {
			cb(null, decoder.write(data));
		});
		return decoder;
	}

C
Christof Marti 已提交
362
	private collectData(stream: Readable): Buffer[] {
363 364
		const buffers: Buffer[] = [];
		stream.on('data', (data: Buffer) => {
C
Christof Marti 已提交
365 366 367 368 369
			buffers.push(data);
		});
		return buffers;
	}

C
Christof Marti 已提交
370 371 372 373 374
	private decodeData(buffers: Buffer[], encoding: string): string {
		const decoder = new StringDecoder(encoding);
		return buffers.map(buffer => decoder.write(buffer)).join('');
	}

375 376 377 378 379 380 381 382 383
	private initDirectoryTree(): IDirectoryTree {
		const tree: IDirectoryTree = {
			rootEntries: [],
			pathToEntries: Object.create(null)
		};
		tree.pathToEntries['.'] = tree.rootEntries;
		return tree;
	}

384
	private addDirectoryEntries({ pathToEntries }: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
385
		this.cmdResultCount += relativeFiles.length;
C
Christof Marti 已提交
386 387 388

		// Support relative paths to files from a root resource (ignores excludes)
		if (relativeFiles.indexOf(this.filePattern) !== -1) {
389
			const basename = path.basename(this.filePattern);
390
			this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
C
Christof Marti 已提交
391 392
		}

393
		function add(relativePath: string) {
394 395
			const basename = path.basename(relativePath);
			const dirname = path.dirname(relativePath);
C
Christof Marti 已提交
396 397 398 399 400 401
			let entries = pathToEntries[dirname];
			if (!entries) {
				entries = pathToEntries[dirname] = [];
				add(dirname);
			}
			entries.push({
402
				base,
C
Christof Marti 已提交
403 404 405
				relativePath,
				basename
			});
406 407
		}
		relativeFiles.forEach(add);
C
Christof Marti 已提交
408 409
	}

410
	private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
C
Christof Marti 已提交
411
		const self = this;
412
		const excludePattern = this.folderExcludePatterns.get(rootFolder);
C
Christof Marti 已提交
413 414 415 416
		const filePattern = this.filePattern;
		function matchDirectory(entries: IDirectoryEntry[]) {
			self.directoriesWalked++;
			for (let i = 0, n = entries.length; i < n; i++) {
417
				const entry = entries[i];
418
				const { relativePath, basename } = entry;
C
Christof Marti 已提交
419 420 421 422 423

				// Check exclude pattern
				// If the user searches for the exact file name, we adjust the glob matching
				// to ignore filtering by siblings because the user seems to know what she
				// is searching for and we want to include the result in that case anyway
424
				if (excludePattern.test(relativePath, basename, () => filePattern !== basename ? entries.map(entry => entry.basename) : [])) {
C
Christof Marti 已提交
425 426
					continue;
				}
E
Erich Gamma 已提交
427

C
Christof Marti 已提交
428 429 430 431 432 433 434 435 436
				const sub = pathToEntries[relativePath];
				if (sub) {
					matchDirectory(sub);
				} else {
					self.filesWalked++;
					if (relativePath === filePattern) {
						continue; // ignore file if its path matches with the file pattern because that is already matched above
					}

437
					self.matchFile(onResult, entry);
C
Christof Marti 已提交
438 439 440 441 442 443
				}
			};
		}
		matchDirectory(rootEntries);
	}

444
	private nodeJSTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
C
Christof Marti 已提交
445
		this.directoriesWalked++;
446
		extfs.readdir(folderQuery.folder, (error: Error, files: string[]) => {
C
Christof Marti 已提交
447 448 449 450 451
			if (error || this.isCanceled || this.isLimitHit) {
				return done();
			}

			// Support relative paths to files from a root resource (ignores excludes)
452
			return this.checkFilePatternRelativeMatch(folderQuery.folder, (match, size) => {
C
Christof Marti 已提交
453 454 455 456 457 458 459 460
				if (this.isCanceled || this.isLimitHit) {
					return done();
				}

				// Report result from file pattern if matching
				if (match) {
					this.resultCount++;
					onResult({
461
						base: folderQuery.folder,
462
						relativePath: this.filePattern,
463
						basename: path.basename(this.filePattern),
C
Christof Marti 已提交
464
						size
465
					});
C
Christof Marti 已提交
466 467
				}

468
				return this.doWalk(folderQuery, '', files, onResult, done);
E
Erich Gamma 已提交
469
			});
470 471 472
		});
	}

473
	public getStats(): IUncachedSearchStats {
C
chrmarti 已提交
474
		return {
475
			fromCache: false,
C
Christof Marti 已提交
476 477
			traversal: Traversal[this.traversal],
			errors: this.errors,
C
chrmarti 已提交
478 479 480
			fileWalkStartTime: this.fileWalkStartTime,
			fileWalkResultTime: Date.now(),
			directoriesWalked: this.directoriesWalked,
481
			filesWalked: this.filesWalked,
C
Christof Marti 已提交
482 483 484 485
			resultCount: this.resultCount,
			cmdForkStartTime: this.cmdForkStartTime,
			cmdForkResultTime: this.cmdForkResultTime,
			cmdResultCount: this.cmdResultCount
C
chrmarti 已提交
486 487 488
		};
	}

489
	private checkFilePatternAbsoluteMatch(clb: (exists: boolean, size?: number) => void): void {
490
		if (!this.filePattern || !path.isAbsolute(this.filePattern)) {
491 492 493
			return clb(false);
		}

494
		return fs.stat(this.filePattern, (error, stat) => {
495
			return clb(!error && !stat.isDirectory(), stat && stat.size); // only existing files
E
Erich Gamma 已提交
496 497 498
		});
	}

499
	private checkFilePatternRelativeMatch(basePath: string, clb: (matchPath: string, size?: number) => void): void {
500
		if (!this.filePattern || path.isAbsolute(this.filePattern)) {
501 502 503
			return clb(null);
		}

504
		const absolutePath = path.join(basePath, this.filePattern);
505

506
		return fs.stat(absolutePath, (error, stat) => {
507
			return clb(!error && !stat.isDirectory() ? absolutePath : null, stat && stat.size); // only existing files
508 509 510
		});
	}

511 512
	private doWalk(folderQuery: IFolderSearch, relativeParentPath: string, files: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error) => void): void {
		const rootFolder = folderQuery.folder;
E
Erich Gamma 已提交
513 514

		// Execute tasks on each file in parallel to optimize throughput
515
		flow.parallel(files, (file: string, clb: (error: Error, result: {}) => void): void => {
E
Erich Gamma 已提交
516 517 518

			// Check canceled
			if (this.isCanceled || this.isLimitHit) {
519
				return clb(null, undefined);
E
Erich Gamma 已提交
520 521 522 523 524 525
			}

			// If the user searches for the exact file name, we adjust the glob matching
			// to ignore filtering by siblings because the user seems to know what she
			// is searching for and we want to include the result in that case anyway
			let siblings = files;
526
			if (this.config.filePattern === file) {
E
Erich Gamma 已提交
527 528 529 530
				siblings = [];
			}

			// Check exclude pattern
531
			let currentRelativePath = relativeParentPath ? [relativeParentPath, file].join(path.sep) : file;
532
			if (this.folderExcludePatterns.get(folderQuery.folder).test(currentRelativePath, file, () => siblings)) {
533
				return clb(null, undefined);
E
Erich Gamma 已提交
534 535
			}

536
			// Use lstat to detect links
537
			let currentAbsolutePath = [rootFolder, currentRelativePath].join(path.sep);
538
			fs.lstat(currentAbsolutePath, (error, lstat) => {
539
				if (error || this.isCanceled || this.isLimitHit) {
540
					return clb(null, undefined);
541
				}
E
Erich Gamma 已提交
542

543
				// If the path is a link, we must instead use fs.stat() to find out if the
544 545 546 547
				// link is a directory or not because lstat will always return the stat of
				// the link which is always a file.
				this.statLinkIfNeeded(currentAbsolutePath, lstat, (error, stat) => {
					if (error || this.isCanceled || this.isLimitHit) {
548
						return clb(null, undefined);
549
					}
E
Erich Gamma 已提交
550

551 552
					// Directory: Follow directories
					if (stat.isDirectory()) {
C
chrmarti 已提交
553
						this.directoriesWalked++;
554

555 556
						// to really prevent loops with links we need to resolve the real path of them
						return this.realPathIfNeeded(currentAbsolutePath, lstat, (error, realpath) => {
557
							if (error || this.isCanceled || this.isLimitHit) {
558
								return clb(null, undefined);
559 560
							}

561
							if (this.walkedPaths[realpath]) {
562
								return clb(null, undefined); // escape when there are cycles (can happen with symlinks)
563
							}
E
Erich Gamma 已提交
564

565 566 567 568 569
							this.walkedPaths[realpath] = true; // remember as walked

							// Continue walking
							return extfs.readdir(currentAbsolutePath, (error: Error, children: string[]): void => {
								if (error || this.isCanceled || this.isLimitHit) {
570
									return clb(null, undefined);
571 572
								}

573
								this.doWalk(folderQuery, currentRelativePath, children, onResult, err => clb(err, undefined));
574 575
							});
						});
576
					}
E
Erich Gamma 已提交
577

578 579
					// File: Check for match on file pattern and include pattern
					else {
C
chrmarti 已提交
580
						this.filesWalked++;
C
Christof Marti 已提交
581
						if (currentRelativePath === this.filePattern) {
582
							return clb(null, undefined); // ignore file if its path matches with the file pattern because checkFilePatternRelativeMatch() takes care of those
583
						}
E
Erich Gamma 已提交
584

585
						if (this.maxFilesize && types.isNumber(stat.size) && stat.size > this.maxFilesize) {
586
							return clb(null, undefined); // ignore file if max file size is hit
587 588
						}

589
						this.matchFile(onResult, { base: rootFolder, relativePath: currentRelativePath, basename: file, size: stat.size });
590 591 592
					}

					// Unwind
593
					return clb(null, undefined);
594
				});
E
Erich Gamma 已提交
595 596 597 598 599 600
			});
		}, (error: Error[]): void => {
			if (error) {
				error = arrays.coalesce(error); // find any error by removing null values first
			}

C
Christof Marti 已提交
601
			return done(error && error.length > 0 ? error[0] : null);
E
Erich Gamma 已提交
602 603 604
		});
	}

605 606
	private matchFile(onResult: (result: IRawFileMatch) => void, candidate: IRawFileMatch): void {
		if (this.isFilePatternMatch(candidate.relativePath) && (!this.includePattern || this.includePattern(candidate.relativePath, candidate.basename))) {
607 608 609 610 611 612 613
			this.resultCount++;

			if (this.maxResults && this.resultCount > this.maxResults) {
				this.isLimitHit = true;
			}

			if (!this.isLimitHit) {
614
				onResult(candidate);
615 616 617 618
			}
		}
	}

B
polish  
Benjamin Pasero 已提交
619
	private isFilePatternMatch(path: string): boolean {
620 621 622

		// Check for search pattern
		if (this.filePattern) {
623 624 625 626
			if (this.filePattern === '*') {
				return true; // support the all-matching wildcard
			}

627
			return scorer.matches(path, this.normalizedFilePatternLowercase);
628 629 630 631 632 633
		}

		// No patterns means we match all
		return true;
	}

634
	private statLinkIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error, stat: fs.Stats) => void): void {
635
		if (lstat.isSymbolicLink()) {
636
			return fs.stat(path, clb); // stat the target the link points to
637 638 639 640 641
		}

		return clb(null, lstat); // not a link, so the stat is already ok for us
	}

642
	private realPathIfNeeded(path: string, lstat: fs.Stats, clb: (error: Error, realpath?: string) => void): void {
643 644 645 646 647
		if (lstat.isSymbolicLink()) {
			return fs.realpath(path, (error, realpath) => {
				if (error) {
					return clb(error);
				}
E
Erich Gamma 已提交
648

649 650 651
				return clb(null, realpath);
			});
		}
E
Erich Gamma 已提交
652

653
		return clb(null, path);
E
Erich Gamma 已提交
654 655 656
	}
}

657
export class Engine implements ISearchEngine<IRawFileMatch> {
658
	private folderQueries: IFolderSearch[];
659
	private extraFiles: string[];
E
Erich Gamma 已提交
660 661 662
	private walker: FileWalker;

	constructor(config: IRawSearch) {
663
		this.folderQueries = config.folderQueries;
664 665
		this.extraFiles = config.extraFiles;

E
Erich Gamma 已提交
666 667 668
		this.walker = new FileWalker(config);
	}

669
	public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
670
		this.walker.walk(this.folderQueries, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => {
C
chrmarti 已提交
671 672 673 674 675
			done(err, {
				limitHit: isLimitHit,
				stats: this.walker.getStats()
			});
		});
E
Erich Gamma 已提交
676 677 678 679 680
	}

	public cancel(): void {
		this.walker.cancel();
	}
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
}

/**
 * This class exists to provide one interface on top of two ParsedExpressions, one for absolute expressions and one for relative expressions.
 * The absolute and relative expressions don't "have" to be kept separate, but this keeps us from having to path.join every single
 * file searched, it's only used for a text search with a searchPath
 */
class AbsoluteAndRelativeParsedExpression {
	private absoluteParsedExpr: glob.ParsedExpression;
	private relativeParsedExpr: glob.ParsedExpression;

	constructor(expr: glob.IExpression, private root: string) {
		this.init(expr);
	}

	/**
	 * Split the IExpression into its absolute and relative components, and glob.parse them separately.
	 */
	private init(expr: glob.IExpression): void {
		let absoluteGlobExpr: glob.IExpression;
		let relativeGlobExpr: glob.IExpression;
702 703 704 705 706
		Object.keys(expr)
			.filter(key => expr[key])
			.forEach(key => {
				if (path.isAbsolute(key)) {
					absoluteGlobExpr = absoluteGlobExpr || glob.getEmptyExpression();
707
					absoluteGlobExpr[key] = expr[key];
708 709
				} else {
					relativeGlobExpr = relativeGlobExpr || glob.getEmptyExpression();
710
					relativeGlobExpr[key] = expr[key];
711 712
				}
			});
713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747

		this.absoluteParsedExpr = absoluteGlobExpr && glob.parse(absoluteGlobExpr, { trimForExclusions: true });
		this.relativeParsedExpr = relativeGlobExpr && glob.parse(relativeGlobExpr, { trimForExclusions: true });
	}

	public test(_path: string, basename?: string, siblingsFn?: () => string[] | TPromise<string[]>): string | TPromise<string> {
		return (this.relativeParsedExpr && this.relativeParsedExpr(_path, basename, siblingsFn)) ||
			(this.absoluteParsedExpr && this.absoluteParsedExpr(path.join(this.root, _path), basename, siblingsFn));
	}

	public getBasenameTerms(): string[] {
		const basenameTerms = [];
		if (this.absoluteParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			basenameTerms.push(...glob.getBasenameTerms(this.relativeParsedExpr));
		}

		return basenameTerms;
	}

	public getPathTerms(): string[] {
		const pathTerms = [];
		if (this.absoluteParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.absoluteParsedExpr));
		}

		if (this.relativeParsedExpr) {
			pathTerms.push(...glob.getPathTerms(this.relativeParsedExpr));
		}

		return pathTerms;
	}
748
}