From 13953dbbbbfac231128683ab74cc038869d82d76 Mon Sep 17 00:00:00 2001 From: roblou Date: Tue, 22 Nov 2016 18:55:29 -0800 Subject: [PATCH] Works slowly, in 1 proc, no progress --- .vscode/launch.json | 16 ++ package.json | 3 +- .../parts/search/common/searchModel.ts | 10 + .../services/search/node/rawSearchService.ts | 2 +- .../services/search/node/searchService.ts | 3 +- .../services/search/node/textSearch.ts | 212 ++++---------- .../search/node/worker/searchWorker.ts | 266 ++++++++++++++++++ .../search/node/worker/searchWorkerApp.ts | 13 + .../search/node/worker/searchWorkerIpc.ts | 36 +++ 9 files changed, 395 insertions(+), 166 deletions(-) create mode 100644 src/vs/workbench/services/search/node/worker/searchWorker.ts create mode 100644 src/vs/workbench/services/search/node/worker/searchWorkerApp.ts create mode 100644 src/vs/workbench/services/search/node/worker/searchWorkerIpc.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 812630a07f5..35fedef5c91 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -43,6 +43,22 @@ "sourceMaps": true, "outFiles": [ "${workspaceRoot}/out/**/*.js" ] }, + { + "type": "node", + "request": "attach", + "name": "Attach to Search Process", + "port": 5877, + "sourceMaps": true, + "outFiles": [ "${workspaceRoot}/out/**/*.js" ] + }, + { + "type": "node", + "request": "attach", + "name": "Attach to Search Worker", + "port": 5878, + "sourceMaps": true, + "outFiles": [ "${workspaceRoot}/out/**/*.js" ] + }, { "type": "extensionHost", "request": "launch", diff --git a/package.json b/package.json index 1f0b04ba54e..e4538c76d7e 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "native-keymap": "0.3.0", "pty.js": "https://github.com/Tyriar/pty.js/tarball/c75c2dcb6dcad83b0cb3ef2ae42d0448fb912642", "semver": "4.3.6", + "v8-profiler": "git+https://github.com/jrieken/v8-profiler.git", "vscode-debugprotocol": "1.14.0", "vscode-textmate": "2.3.1", "winreg": "1.2.0", @@ -114,4 +115,4 @@ "windows-mutex": "^0.2.0", "fsevents": "0.3.8" } -} \ No newline at end of file +} diff --git a/src/vs/workbench/parts/search/common/searchModel.ts b/src/vs/workbench/parts/search/common/searchModel.ts index 8658e5be72f..eef464f659a 100644 --- a/src/vs/workbench/parts/search/common/searchModel.ts +++ b/src/vs/workbench/parts/search/common/searchModel.ts @@ -26,6 +26,8 @@ import { IReplaceService } from 'vs/workbench/parts/search/common/replace'; import { IProgressRunner } from 'vs/platform/progress/common/progress'; import { RangeHighlightDecorations } from 'vs/workbench/common/editor/rangeDecorations'; +import * as cp from 'child_process'; + export class Match { private _lineText: string; @@ -533,6 +535,14 @@ export class SearchModel extends Disposable { } public search(query: ISearchQuery): PPromise { + // console.log('forking searchWorker'); + // const proc = cp.fork('/Users/roblou/code/vscode/src/searchWorker.js'); + // proc.on('message', m => { + // console.log('parent got message: ' + JSON.stringify(m)); + // }) + + // proc.send({ hello: 'ping' }) + this.cancelSearch(); this.searchResult.clear(); diff --git a/src/vs/workbench/services/search/node/rawSearchService.ts b/src/vs/workbench/services/search/node/rawSearchService.ts index 4ba1f5fb878..f074a6480db 100644 --- a/src/vs/workbench/services/search/node/rawSearchService.ts +++ b/src/vs/workbench/services/search/node/rawSearchService.ts @@ -27,7 +27,7 @@ export type IRawProgressItem = T | T[] | IProgress; export class SearchService implements IRawSearchService { - private static BATCH_SIZE = 512; + private static BATCH_SIZE = 20; private caches: { [cacheKey: string]: Cache; } = Object.create(null); diff --git a/src/vs/workbench/services/search/node/searchService.ts b/src/vs/workbench/services/search/node/searchService.ts index 123d2bc5497..809c0d822c1 100644 --- a/src/vs/workbench/services/search/node/searchService.ts +++ b/src/vs/workbench/services/search/node/searchService.ts @@ -215,7 +215,8 @@ export class DiskSearch { AMD_ENTRYPOINT: 'vs/workbench/services/search/node/searchApp', PIPE_LOGGING: 'true', VERBOSE_LOGGING: verboseLogging - } + }, + // debugBrk: 5877 } ); diff --git a/src/vs/workbench/services/search/node/textSearch.ts b/src/vs/workbench/services/search/node/textSearch.ts index 873cad46a94..85ebde044a3 100644 --- a/src/vs/workbench/services/search/node/textSearch.ts +++ b/src/vs/workbench/services/search/node/textSearch.ts @@ -6,21 +6,19 @@ 'use strict'; import * as strings from 'vs/base/common/strings'; +import uri from 'vs/base/common/uri'; import * as fs from 'fs'; import * as path from 'path'; +import * as cp from 'child_process'; import * as baseMime from 'vs/base/common/mime'; import { ILineMatch, IProgress } from 'vs/platform/search/common/search'; -import { detectMimeAndEncodingFromBuffer } from 'vs/base/node/mime'; import { FileWalker } from 'vs/workbench/services/search/node/fileSearch'; import { UTF16le, UTF16be, UTF8, UTF8_with_bom, encodingExists, decode } from 'vs/base/node/encoding'; import { ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine } from './search'; -interface ReadLinesOptions { - bufferLength: number; - encoding: string; -} +import { Client } from 'vs/base/parts/ipc/node/ipc.cp'; export class Engine implements ISearchEngine { @@ -30,7 +28,7 @@ export class Engine implements ISearchEngine { private extraFiles: string[]; private maxResults: number; private walker: FileWalker; - private contentPattern: RegExp; + private contentPattern: string; private isCanceled: boolean; private isDone: boolean; private total: number; @@ -41,11 +39,19 @@ export class Engine implements ISearchEngine { private fileEncoding: string; private limitReached: boolean; + // private worker: cp.ChildProcess; + private client: Client; + private channel: any; + + private onResult: any; + constructor(config: IRawSearch, walker: FileWalker) { this.rootFolders = config.rootFolders; this.extraFiles = config.extraFiles; this.walker = walker; - this.contentPattern = strings.createRegExp(config.contentPattern.pattern, config.contentPattern.isRegExp, { matchCase: config.contentPattern.isCaseSensitive, wholeWord: config.contentPattern.isWordMatch, multiline: false, global: true }); + this.contentPattern = config.contentPattern.pattern; + const pattern = strings.createRegExp(config.contentPattern.pattern, config.contentPattern.isRegExp, { matchCase: config.contentPattern.isCaseSensitive, wholeWord: config.contentPattern.isWordMatch, multiline: false, global: true }); + console.log('pattern: ' + pattern.toString()); this.isCanceled = false; this.limitReached = false; this.maxResults = config.maxResults; @@ -53,6 +59,33 @@ export class Engine implements ISearchEngine { this.progressed = 0; this.total = 0; this.fileEncoding = encodingExists(config.fileEncoding) ? config.fileEncoding : UTF8; + + // this.worker = cp.fork('/Users/roblou/code/vscode/out/vs/workbench/services/search/node/searchWorker.js', [], { execArgv: ['--debug-brk=5878']}); + // this.worker.on('message', m => { + // console.log('parent got message'); + // if (this.onResult) { + // this.onResult(JSON.parse(m)); + // } + // }); + + // this.worker.send({ initialize: { contentPattern: config.contentPattern.pattern }}); + this.client = new Client( + uri.parse(require.toUrl('bootstrap')).fsPath, + { + serverName: 'Search Worker', + timeout: 60 * 60 * 1000, + args: ['--type=searchWorker'], + env: { + AMD_ENTRYPOINT: 'vs/workbench/services/search/node/worker/searchWorkerApp', + PIPE_LOGGING: 'true', + VERBOSE_LOGGING: 'true' + }, + // debugBrk: 5878 + } + ); + this.channel = this.client.getChannel('searchWorker'); + + // process.on('exit', () => this.worker.kill()); } public cancel(): void { @@ -61,7 +94,10 @@ export class Engine implements ISearchEngine { } public search(onResult: (match: ISerializedFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void { + this.channel.call('initialize', { contentPattern: this.contentPattern }); + let resultCounter = 0; + this.onResult = onResult; let progress = () => { this.progressed++; @@ -101,171 +137,21 @@ export class Engine implements ISearchEngine { // Indicate progress to the outside progress(); - let fileMatch: FileMatch = null; - - let doneCallback = (error?: Error) => { - if (!error && !this.isCanceled && fileMatch && !fileMatch.isEmpty()) { - onResult(fileMatch.serialize()); - } - - return unwind(size); - }; - const absolutePath = result.base ? [result.base, result.relativePath].join(path.sep) : result.relativePath; - let perLineCallback = (line: string, lineNumber: number) => { - if (this.limitReached || this.isCanceled) { - return; // return early if canceled or limit reached + this.channel.call('search', absolutePath).then(fileMatch => { + // console.log('got result: ' + fileMatch); + if (fileMatch && fileMatch.lineMatches.length) { + onResult(fileMatch); } - let lineMatch: LineMatch = null; - let match = this.contentPattern.exec(line); - - // Record all matches into file result - while (match !== null && match[0].length > 0 && !this.limitReached && !this.isCanceled) { - resultCounter++; - if (this.maxResults && resultCounter >= this.maxResults) { - this.limitReached = true; - } - - if (fileMatch === null) { - fileMatch = new FileMatch(absolutePath); - } - - if (lineMatch === null) { - lineMatch = new LineMatch(line, lineNumber); - fileMatch.addMatch(lineMatch); - } - - lineMatch.addMatch(match.index, match[0].length); - - match = this.contentPattern.exec(line); - } - }; - - // Read lines buffered to support large files - this.readlinesAsync(absolutePath, perLineCallback, { bufferLength: 8096, encoding: this.fileEncoding }, doneCallback); + unwind(size); + }); }, (error, isLimitHit) => { this.walkerIsDone = true; this.walkerError = error; unwind(0 /* walker is done, indicate this back to our handler to be able to unwind */); }); } - - private readlinesAsync(filename: string, perLineCallback: (line: string, lineNumber: number) => void, options: ReadLinesOptions, callback: (error: Error) => void): void { - fs.open(filename, 'r', null, (error: Error, fd: number) => { - if (error) { - return callback(error); - } - - let buffer = new Buffer(options.bufferLength); - let pos: number; - let i: number; - let line = ''; - let lineNumber = 0; - let lastBufferHadTraillingCR = false; - - const outer = this; - - function decodeBuffer(buffer: NodeBuffer): string { - if (options.encoding === UTF8 || options.encoding === UTF8_with_bom) { - return buffer.toString(); // much faster to use built in toString() when encoding is default - } - - return decode(buffer, options.encoding); - } - - function lineFinished(offset: number): void { - line += decodeBuffer(buffer.slice(pos, i + offset)); - perLineCallback(line, lineNumber); - line = ''; - lineNumber++; - pos = i + offset; - } - - function readFile(isFirstRead: boolean, clb: (error: Error) => void): void { - if (outer.limitReached || outer.isCanceled) { - return clb(null); // return early if canceled or limit reached - } - - fs.read(fd, buffer, 0, buffer.length, null, (error: Error, bytesRead: number, buffer: NodeBuffer) => { - if (error || bytesRead === 0 || outer.limitReached || outer.isCanceled) { - return clb(error); // return early if canceled or limit reached or no more bytes to read - } - - pos = 0; - i = 0; - - // Detect encoding and mime when this is the beginning of the file - if (isFirstRead) { - let mimeAndEncoding = detectMimeAndEncodingFromBuffer(buffer, bytesRead); - if (mimeAndEncoding.mimes[mimeAndEncoding.mimes.length - 1] !== baseMime.MIME_TEXT) { - return clb(null); // skip files that seem binary - } - - // Check for BOM offset - switch (mimeAndEncoding.encoding) { - case UTF8: - pos = i = 3; - options.encoding = UTF8; - break; - case UTF16be: - pos = i = 2; - options.encoding = UTF16be; - break; - case UTF16le: - pos = i = 2; - options.encoding = UTF16le; - break; - } - } - - if (lastBufferHadTraillingCR) { - if (buffer[i] === 0x0a) { // LF (Line Feed) - lineFinished(1); - i++; - } else { - lineFinished(0); - } - - lastBufferHadTraillingCR = false; - } - - for (; i < bytesRead; ++i) { - if (buffer[i] === 0x0a) { // LF (Line Feed) - lineFinished(1); - } else if (buffer[i] === 0x0d) { // CR (Carriage Return) - if (i + 1 === bytesRead) { - lastBufferHadTraillingCR = true; - } else if (buffer[i + 1] === 0x0a) { // LF (Line Feed) - lineFinished(2); - i++; - } else { - lineFinished(1); - } - } - } - - line += decodeBuffer(buffer.slice(pos, bytesRead)); - - readFile(false /* isFirstRead */, clb); // Continue reading - }); - } - - readFile(true /* isFirstRead */, (error: Error) => { - if (error) { - return callback(error); - } - - if (line.length) { - perLineCallback(line, lineNumber); // handle last line - } - - fs.close(fd, (error: Error) => { - callback(error); - }); - }); - }); - } } class FileMatch implements ISerializedFileMatch { diff --git a/src/vs/workbench/services/search/node/worker/searchWorker.ts b/src/vs/workbench/services/search/node/worker/searchWorker.ts new file mode 100644 index 00000000000..4b5fa83cc9e --- /dev/null +++ b/src/vs/workbench/services/search/node/worker/searchWorker.ts @@ -0,0 +1,266 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import * as fs from 'fs'; + +import * as strings from 'vs/base/common/strings'; +import { PPromise, TPromise } from 'vs/base/common/winjs.base'; +import { ISerializedFileMatch } from '../search'; +import * as baseMime from 'vs/base/common/mime'; +import { ILineMatch } from 'vs/platform/search/common/search'; +import { UTF16le, UTF16be, UTF8, UTF8_with_bom, encodingExists, decode } from 'vs/base/node/encoding'; +import { detectMimeAndEncodingFromBuffer } from 'vs/base/node/mime'; + +import profiler = require('v8-profiler'); + +interface ReadLinesOptions { + bufferLength: number; + encoding: string; +} + +// let worker: SearchWorker; +// process.on('message', m => { +// if (m.initialize) { +// worker = new SearchWorker(m.initialize); +// } else { +// worker.search(m.absolutePath); +// } +// }) + +export class SearchWorker { + private contentPattern: RegExp; + + private limitReached: boolean; + private isCanceled: boolean; + + private nResults = 0; + + constructor(args: any) { + this.contentPattern = strings.createRegExp(args.contentPattern, false, { multiline: false, global: true, matchCase: false }); + profiler.startProfiling('p1'); + } + + public search(absolutePath: string): TPromise { + + let fileMatch: FileMatch = null; + // console.log('doing search: ' + absolutePath); + + let perLineCallback = (line: string, lineNumber: number) => { + let lineMatch: LineMatch = null; + let match = this.contentPattern.exec(line); + + // Record all matches into file result + while (match !== null && match[0].length > 0 && !this.limitReached && !this.isCanceled) { + if (fileMatch === null) { + fileMatch = new FileMatch(absolutePath); + } + + if (lineMatch === null) { + lineMatch = new LineMatch(line, lineNumber); + fileMatch.addMatch(lineMatch); + } + + lineMatch.addMatch(match.index, match[0].length); + + match = this.contentPattern.exec(line); + } + }; + + return new TPromise(resolve => { + // Read lines buffered to support large files + this.readlinesAsync(absolutePath, perLineCallback, { bufferLength: 8096, encoding: 'utf8' }, resolve); + }).then(() => { + if (this.nResults++ === 100) { + const p1 = profiler.stopProfiling('p1'); + p1.export() + .pipe(fs.createWriteStream('/Users/roblou/code/vscode/p1.cpuprofile')) + .on('finish', () => p1.delete()); + } + return fileMatch; + }); + } + + private readlinesAsync(filename: string, perLineCallback: (line: string, lineNumber: number) => void, options: ReadLinesOptions, callback: (error: Error) => void): void { + fs.open(filename, 'r', null, (error: Error, fd: number) => { + if (error) { + return callback(error); + } + + let buffer = new Buffer(options.bufferLength); + let pos: number; + let i: number; + let line = ''; + let lineNumber = 0; + let lastBufferHadTraillingCR = false; + + const decodeBuffer = (buffer: NodeBuffer, start, end): string => { + if (options.encoding === UTF8 || options.encoding === UTF8_with_bom) { + return buffer.toString(undefined, start, end); // much faster to use built in toString() when encoding is default + } + + return decode(buffer.slice(start, end), options.encoding); + }; + + const lineFinished = (offset: number): void => { + line += decodeBuffer(buffer, pos, i + offset); + perLineCallback(line, lineNumber); + line = ''; + lineNumber++; + pos = i + offset; + }; + + const readFile = (isFirstRead: boolean, clb: (error: Error) => void): void => { + if (this.limitReached || this.isCanceled) { + return clb(null); // return early if canceled or limit reached + } + + fs.read(fd, buffer, 0, buffer.length, null, (error: Error, bytesRead: number, buffer: NodeBuffer) => { + if (error || bytesRead === 0 || this.limitReached || this.isCanceled) { + return clb(error); // return early if canceled or limit reached or no more bytes to read + } + + pos = 0; + i = 0; + + // Detect encoding and mime when this is the beginning of the file + if (isFirstRead) { + let mimeAndEncoding = detectMimeAndEncodingFromBuffer(buffer, bytesRead); + if (mimeAndEncoding.mimes[mimeAndEncoding.mimes.length - 1] !== baseMime.MIME_TEXT) { + return clb(null); // skip files that seem binary + } + + // Check for BOM offset + switch (mimeAndEncoding.encoding) { + case UTF8: + pos = i = 3; + options.encoding = UTF8; + break; + case UTF16be: + pos = i = 2; + options.encoding = UTF16be; + break; + case UTF16le: + pos = i = 2; + options.encoding = UTF16le; + break; + } + } + + if (lastBufferHadTraillingCR) { + if (buffer[i] === 0x0a) { // LF (Line Feed) + lineFinished(1); + i++; + } else { + lineFinished(0); + } + + lastBufferHadTraillingCR = false; + } + + for (; i < bytesRead; ++i) { + if (buffer[i] === 0x0a) { // LF (Line Feed) + lineFinished(1); + } else if (buffer[i] === 0x0d) { // CR (Carriage Return) + if (i + 1 === bytesRead) { + lastBufferHadTraillingCR = true; + } else if (buffer[i + 1] === 0x0a) { // LF (Line Feed) + lineFinished(2); + i++; + } else { + lineFinished(1); + } + } + } + + line += decodeBuffer(buffer, pos, bytesRead); + + readFile(false /* isFirstRead */, clb); // Continue reading + }); + } + + readFile(true /* isFirstRead */, (error: Error) => { + if (error) { + return callback(error); + } + + if (line.length) { + perLineCallback(line, lineNumber); // handle last line + } + + fs.close(fd, (error: Error) => { + callback(error); + }); + }); + }); + } +} + + +export class FileMatch implements ISerializedFileMatch { + public path: string; + public lineMatches: LineMatch[]; + + constructor(path: string) { + this.path = path; + this.lineMatches = []; + } + + public addMatch(lineMatch: LineMatch): void { + this.lineMatches.push(lineMatch); + } + + public isEmpty(): boolean { + return this.lineMatches.length === 0; + } + + public serialize(): ISerializedFileMatch { + let lineMatches: ILineMatch[] = []; + + for (let i = 0; i < this.lineMatches.length; i++) { + lineMatches.push(this.lineMatches[i].serialize()); + } + + return { + path: this.path, + lineMatches: lineMatches + }; + } +} + +export class LineMatch implements ILineMatch { + public preview: string; + public lineNumber: number; + public offsetAndLengths: number[][]; + + constructor(preview: string, lineNumber: number) { + this.preview = preview.replace(/(\r|\n)*$/, ''); + this.lineNumber = lineNumber; + this.offsetAndLengths = []; + } + + public getText(): string { + return this.preview; + } + + public getLineNumber(): number { + return this.lineNumber; + } + + public addMatch(offset: number, length: number): void { + this.offsetAndLengths.push([offset, length]); + } + + public serialize(): ILineMatch { + let result = { + preview: this.preview, + lineNumber: this.lineNumber, + offsetAndLengths: this.offsetAndLengths + }; + + return result; + } +} \ No newline at end of file diff --git a/src/vs/workbench/services/search/node/worker/searchWorkerApp.ts b/src/vs/workbench/services/search/node/worker/searchWorkerApp.ts new file mode 100644 index 00000000000..667a344eac2 --- /dev/null +++ b/src/vs/workbench/services/search/node/worker/searchWorkerApp.ts @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { Server } from 'vs/base/parts/ipc/node/ipc.cp'; +import { SearchWorkerChannel } from './searchWorkerIpc'; + +const server = new Server(); +const channel = new SearchWorkerChannel(); +server.registerChannel('searchWorker', channel); diff --git a/src/vs/workbench/services/search/node/worker/searchWorkerIpc.ts b/src/vs/workbench/services/search/node/worker/searchWorkerIpc.ts new file mode 100644 index 00000000000..c9aba88b52f --- /dev/null +++ b/src/vs/workbench/services/search/node/worker/searchWorkerIpc.ts @@ -0,0 +1,36 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { PPromise, TPromise } from 'vs/base/common/winjs.base'; +import { IChannel } from 'vs/base/parts/ipc/common/ipc'; +import { IRawSearch, ISerializedSearchComplete, ISerializedSearchProgressItem } from '../search'; +import { SearchWorker } from './searchWorker' + +// export interface ISearchWorkerChannel extends IChannel { +// call(command: 'initialize', args: any): TPromise; +// call(command: 'ping'): TPromise; +// call(command: 'search', absolutePath: string): PPromise; +// call(command: string, arg: any): TPromise; +// } + +export class SearchWorkerChannel implements IChannel { + private worker: SearchWorker; + + constructor() { + } + + call(command: string, arg: any): TPromise { + if (command === 'initialize') { + this.worker = new SearchWorker(arg); + return TPromise.wrap(null); + } else if (command === 'ping') { + return TPromise.wrap('pong'); + } else if (command === 'search') { + return this.worker.search(arg); + } + } +} -- GitLab