Incrementally decode find result (fixes #11842)

src/vs/workbench/services/search/node/fileSearch.ts

@@ -6,7 +6,7 @@
 'use strict';
 
 import * as childProcess from 'child_process';
-import {StringDecoder} from 'string_decoder';
+import {StringDecoder, NodeStringDecoder} from 'string_decoder';
 import {toErrorMessage} from 'vs/base/common/errorMessage';
 import fs = require('fs');
 import paths = require('path');
@@ -129,15 +129,14 @@ export class FileWalker {
 			if (!this.maxFilesize) {
 				if (platform.isMacintosh) {
 					this.traversal = Traversal.MacFind;
-					traverse = this.macFindTraversal;
+					traverse = this.findTraversal;
 				// Disable 'dir' for now (#11181, #11179, #11183, #11182).
-				// TS (2.0.2) warns about unreachable code. Using comments.
-				} /* else if (false && platform.isWindows) {
+				} /* else if (platform.isWindows) {
 					this.traversal = Traversal.WindowsDir;
 					traverse = this.windowsDirTraversal;
 				} */ else if (platform.isLinux) {
 					this.traversal = Traversal.LinuxFind;
-					traverse = this.linuxFindTraversal;
+					traverse = this.findTraversal;
 				}
 			}
 
@@ -169,48 +168,38 @@ export class FileWalker {
 		});
 	}
 
-	private macFindTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
+	private findTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
+		const isMac = platform.isMacintosh;
+		let done = (err?: Error) => {
+			done = () => {};
+			cb(err);
+		};
+		let leftover = '';
+		let first = true;
+		const tree = this.initDirectoryTree();
 		const cmd = this.spawnFindCmd(rootFolder, this.excludePattern);
-		this.readStdout(cmd, 'utf8', (err: Error, stdout?: string) => {
+		this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => {
 			if (err) {
 				done(err);
 				return;
 			}
 
 			// Mac: uses NFD unicode form on disk, but we want NFC
-			const relativeFiles = strings.normalizeNFC(stdout).split('\n./');
-			relativeFiles[0] = relativeFiles[0].trim().substr(2);
-			const n = relativeFiles.length;
-			relativeFiles[n - 1] = relativeFiles[n - 1].trim();
-			if (!relativeFiles[n - 1]) {
-				relativeFiles.pop();
-			}
-
-			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
-				done(new Error('Splitting up files failed'));
-				return;
-			}
-
-			this.matchFiles(rootFolder, relativeFiles, onResult);
-
-			done();
-		});
-	}
-
-	protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
-		const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
-		this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
-			if (err) {
-				done(err);
-				return;
+			const normalized = leftover + (isMac ? strings.normalizeNFC(stdout) : stdout);
+			const relativeFiles = normalized.split('\n./');
+			if (first && normalized.length >= 2) {
+				first = false;
+				relativeFiles[0] = relativeFiles[0].trim().substr(2);
 			}
 
-			const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
-			relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
-			const n = relativeFiles.length;
-			relativeFiles[n - 1] = relativeFiles[n - 1].trim();
-			if (!relativeFiles[n - 1]) {
-				relativeFiles.pop();
+			if (last) {
+				const n = relativeFiles.length;
+				relativeFiles[n - 1] = relativeFiles[n - 1].trim();
+				if (!relativeFiles[n - 1]) {
+					relativeFiles.pop();
+				}
+			} else {
+				leftover = relativeFiles.pop();
 			}
 
 			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
@@ -218,39 +207,42 @@ export class FileWalker {
 				return;
 			}
 
-			this.matchFiles(rootFolder, relativeFiles, onResult);
-
-			done();
-		});
-	}
-
-	private linuxFindTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
-		const cmd = this.spawnFindCmd(rootFolder, this.excludePattern);
-		this.readStdout(cmd, 'utf8', (err: Error, stdout?: string) => {
-			if (err) {
-				done(err);
-				return;
-			}
-
-			const relativeFiles = stdout.split('\n./');
-			relativeFiles[0] = relativeFiles[0].trim().substr(2);
-			const n = relativeFiles.length;
-			relativeFiles[n - 1] = relativeFiles[n - 1].trim();
-			if (!relativeFiles[n - 1]) {
-				relativeFiles.pop();
-			}
+			this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
 
-			if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
-				done(new Error('Splitting up files failed'));
-				return;
+			if (last) {
+				this.matchDirectoryTree(tree, rootFolder, onResult);
+				done();
 			}
-
-			this.matchFiles(rootFolder, relativeFiles, onResult);
-
-			done();
 		});
 	}
 
+	// protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
+	// 	const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
+	// 	this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
+	// 		if (err) {
+	// 			done(err);
+	// 			return;
+	// 		}
+
+	// 		const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
+	// 		relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
+	// 		const n = relativeFiles.length;
+	// 		relativeFiles[n - 1] = relativeFiles[n - 1].trim();
+	// 		if (!relativeFiles[n - 1]) {
+	// 			relativeFiles.pop();
+	// 		}
+
+	// 		if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
+	// 			done(new Error('Splitting up files failed'));
+	// 			return;
+	// 		}
+
+	// 		this.matchFiles(rootFolder, relativeFiles, onResult);
+
+	// 		done();
+	// 	});
+	// }
+
 	/**
 	 * Public for testing.
 	 */
@@ -279,13 +271,30 @@ export class FileWalker {
 	 * Public for testing.
 	 */
 	public readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string) => void): void {
-		let done = (err: Error, stdout?: string) => {
-			done = () => {};
-			this.cmdForkResultTime = Date.now();
-			cb(err, stdout);
+		let all = '';
+		this.collectStdout(cmd, encoding, (err: Error, stdout?: string, last?: boolean) => {
+			if (err) {
+				cb(err);
+				return;
+			}
+
+			all += stdout;
+			if (last) {
+				cb(null, all);
+			}
+		});
+	}
+
+	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
+		let done = (err: Error, stdout?: string, last?: boolean) => {
+			if (err || last) {
+				done = () => {};
+				this.cmdForkResultTime = Date.now();
+			}
+			cb(err, stdout, last);
 		};
 
-		const stdout = this.collectData(cmd.stdout);
+		this.forwardData(cmd.stdout, encoding, done);
 		const stderr = this.collectData(cmd.stderr);
 
 		cmd.on('error', (err: Error) => {
@@ -296,11 +305,19 @@ export class FileWalker {
 			if (code !== 0) {
 				done(new Error(`find failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
 			} else {
-				done(null, this.decodeData(stdout, encoding));
+				done(null, '', true);
 			}
 		});
 	}
 
+	private forwardData(stream: Readable, encoding: string, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
+		const decoder = new StringDecoder(encoding);
+		stream.on('data', (data: Buffer) => {
+			cb(null, decoder.write(data));
+		});
+		return decoder;
+	}
+
 	private collectData(stream: Readable): Buffer[] {
 		const buffers: Buffer[] = [];
 		stream.on('data', (data: Buffer) => {
@@ -314,27 +331,25 @@ export class FileWalker {
 		return buffers.map(buffer => decoder.write(buffer)).join('');
 	}
 
-	private matchFiles(rootFolder: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
-		this.cmdResultCount = relativeFiles.length;
+	private initDirectoryTree(): IDirectoryTree {
+		const tree: IDirectoryTree = {
+			rootEntries: [],
+			pathToEntries: Object.create(null)
+		};
+		tree.pathToEntries['.'] = tree.rootEntries;
+		return tree;
+	}
+
+	private addDirectoryEntries({pathToEntries}: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
+		this.cmdResultCount += relativeFiles.length;
 
 		// Support relative paths to files from a root resource (ignores excludes)
 		if (relativeFiles.indexOf(this.filePattern) !== -1) {
 			const basename = paths.basename(this.filePattern);
-			this.matchFile(onResult, { base: rootFolder, relativePath: this.filePattern, basename });
+			this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
 		}
 
-		const tree = this.buildDirectoryTree(rootFolder, relativeFiles);
-		this.matchDirectoryTree(rootFolder, tree, onResult);
-	}
-
-	private buildDirectoryTree(base: string, relativeFilePaths: string[]): IDirectoryTree {
-		const tree: IDirectoryTree = {
-			rootEntries: [],
-			pathToEntries: Object.create(null)
-		};
-		const {pathToEntries} = tree;
-		pathToEntries['.'] = tree.rootEntries;
-		relativeFilePaths.forEach(function add(relativePath: string) {
+		relativeFiles.forEach(function add(relativePath: string) {
 			const basename = paths.basename(relativePath);
 			const dirname = paths.dirname(relativePath);
 			let entries = pathToEntries[dirname];
@@ -348,10 +363,9 @@ export class FileWalker {
 				basename
 			});
 		});
-		return tree;
 	}
 
-	private matchDirectoryTree(rootFolder: string, { rootEntries, pathToEntries }: IDirectoryTree, onResult: (result: IRawFileMatch) => void) {
+	private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
 		const self = this;
 		const excludePattern = this.excludePattern;
 		const filePattern = this.filePattern;