From d616c781e0ebb9930486c29fe6c866440591160b Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Mon, 15 Oct 2018 15:47:11 -0700 Subject: [PATCH] Refactor search - remove search-rg text impl --- extensions/search-rg/src/extension.ts | 9 +- extensions/search-rg/src/ripgrepFileSearch.ts | 32 +- extensions/search-rg/src/ripgrepTextSearch.ts | 376 ------------------ 3 files changed, 32 insertions(+), 385 deletions(-) delete mode 100644 extensions/search-rg/src/ripgrepTextSearch.ts diff --git a/extensions/search-rg/src/extension.ts b/extensions/search-rg/src/extension.ts index 8b4e9963e6e..2621cc64757 100644 --- a/extensions/search-rg/src/extension.ts +++ b/extensions/search-rg/src/extension.ts @@ -5,7 +5,6 @@ import * as vscode from 'vscode'; import { RipgrepFileSearchEngine } from './ripgrepFileSearch'; -import { RipgrepTextSearchEngine } from './ripgrepTextSearch'; export function activate(): void { if (vscode.workspace.getConfiguration('searchRipgrep').get('enable')) { @@ -13,22 +12,16 @@ export function activate(): void { const provider = new RipgrepSearchProvider(outputChannel); vscode.workspace.registerFileIndexProvider('file', provider); - vscode.workspace.registerTextSearchProvider('file', provider); } } -class RipgrepSearchProvider implements vscode.FileIndexProvider, vscode.TextSearchProvider { +class RipgrepSearchProvider implements vscode.FileIndexProvider { private inProgress: Set = new Set(); constructor(private outputChannel: vscode.OutputChannel) { process.once('exit', () => this.dispose()); } - provideTextSearchResults(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, progress: vscode.Progress, token: vscode.CancellationToken): Promise { - const engine = new RipgrepTextSearchEngine(this.outputChannel); - return this.withToken(token, token => engine.provideTextSearchResults(query, options, progress, token)); - } - provideFileIndex(options: vscode.FileSearchOptions, token: vscode.CancellationToken): Thenable { const engine = new RipgrepFileSearchEngine(this.outputChannel); diff --git a/extensions/search-rg/src/ripgrepFileSearch.ts b/extensions/search-rg/src/ripgrepFileSearch.ts index 313d641a617..a8bd5d086e0 100644 --- a/extensions/search-rg/src/ripgrepFileSearch.ts +++ b/extensions/search-rg/src/ripgrepFileSearch.ts @@ -9,7 +9,6 @@ import { NodeStringDecoder, StringDecoder } from 'string_decoder'; import * as vscode from 'vscode'; import { normalizeNFC, normalizeNFD } from './normalization'; import { rgPath } from './ripgrep'; -import { rgErrorMsgForDisplay } from './ripgrepTextSearch'; import { anchorGlob, Maybe } from './utils'; const isMac = process.platform === 'darwin'; @@ -210,3 +209,34 @@ function getRgArgs(options: vscode.FileSearchOptions): string[] { return args; } + +/** + * Read the first line of stderr and return an error for display or undefined, based on a whitelist. + * Ripgrep produces stderr output which is not from a fatal error, and we only want the search to be + * "failed" when a fatal error was produced. + */ +export function rgErrorMsgForDisplay(msg: string): Maybe { + const firstLine = msg.split('\n')[0].trim(); + + if (firstLine.startsWith('Error parsing regex')) { + return firstLine; + } + + if (firstLine.startsWith('error parsing glob') || + firstLine.startsWith('unsupported encoding')) { + // Uppercase first letter + return firstLine.charAt(0).toUpperCase() + firstLine.substr(1); + } + + if (firstLine === `Literal '\\n' not allowed.`) { + // I won't localize this because none of the Ripgrep error messages are localized + return `Literal '\\n' currently not supported`; + } + + if (firstLine.startsWith('Literal ')) { + // Other unsupported chars + return firstLine; + } + + return undefined; +} diff --git a/extensions/search-rg/src/ripgrepTextSearch.ts b/extensions/search-rg/src/ripgrepTextSearch.ts deleted file mode 100644 index 7d550e97353..00000000000 --- a/extensions/search-rg/src/ripgrepTextSearch.ts +++ /dev/null @@ -1,376 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as cp from 'child_process'; -import { EventEmitter } from 'events'; -import * as path from 'path'; -import { NodeStringDecoder, StringDecoder } from 'string_decoder'; -import * as vscode from 'vscode'; -import { rgPath } from './ripgrep'; -import { anchorGlob, createTextSearchResult, Maybe } from './utils'; - -// If vscode-ripgrep is in an .asar file, then the binary is unpacked. -const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked'); - -export class RipgrepTextSearchEngine { - constructor(private outputChannel: vscode.OutputChannel) { } - - provideTextSearchResults(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, progress: vscode.Progress, token: vscode.CancellationToken): Thenable { - this.outputChannel.appendLine(`provideTextSearchResults ${query.pattern}, ${JSON.stringify({ - ...options, - ...{ - folder: options.folder.toString() - } - })}`); - - return new Promise((resolve, reject) => { - token.onCancellationRequested(() => cancel()); - - const rgArgs = getRgArgs(query, options); - - const cwd = options.folder.fsPath; - - const escapedArgs = rgArgs - .map(arg => arg.match(/^-/) ? arg : `'${arg}'`) - .join(' '); - this.outputChannel.appendLine(`rg ${escapedArgs}\n - cwd: ${cwd}`); - - let rgProc: Maybe = cp.spawn(rgDiskPath, rgArgs, { cwd }); - rgProc.on('error', e => { - console.error(e); - this.outputChannel.append('Error: ' + (e && e.message)); - reject(e); - }); - - let gotResult = false; - const ripgrepParser = new RipgrepParser(options.maxResults, cwd, options.previewOptions); - ripgrepParser.on('result', (match: vscode.TextSearchResult) => { - gotResult = true; - progress.report(match); - }); - - let isDone = false; - const cancel = () => { - isDone = true; - - if (rgProc) { - rgProc.kill(); - } - - if (ripgrepParser) { - ripgrepParser.cancel(); - } - }; - - let limitHit = false; - ripgrepParser.on('hitLimit', () => { - limitHit = true; - cancel(); - }); - - rgProc.stdout.on('data', data => { - ripgrepParser.handleData(data); - }); - - let gotData = false; - rgProc.stdout.once('data', () => gotData = true); - - let stderr = ''; - rgProc.stderr.on('data', data => { - const message = data.toString(); - this.outputChannel.append(message); - stderr += message; - }); - - rgProc.on('close', () => { - this.outputChannel.appendLine(gotData ? 'Got data from stdout' : 'No data from stdout'); - this.outputChannel.appendLine(gotResult ? 'Got result from parser' : 'No result from parser'); - this.outputChannel.appendLine(''); - if (isDone) { - resolve({ limitHit }); - } else { - // Trigger last result - ripgrepParser.flush(); - rgProc = null; - let displayMsg: Maybe; - if (stderr && !gotData && (displayMsg = rgErrorMsgForDisplay(stderr))) { - reject(new Error(displayMsg)); - } else { - resolve({ limitHit }); - } - } - }); - }); - } -} - -/** - * Read the first line of stderr and return an error for display or undefined, based on a whitelist. - * Ripgrep produces stderr output which is not from a fatal error, and we only want the search to be - * "failed" when a fatal error was produced. - */ -export function rgErrorMsgForDisplay(msg: string): Maybe { - const firstLine = msg.split('\n')[0].trim(); - - if (firstLine.startsWith('Error parsing regex')) { - return firstLine; - } - - if (firstLine.startsWith('error parsing glob') || - firstLine.startsWith('unsupported encoding')) { - // Uppercase first letter - return firstLine.charAt(0).toUpperCase() + firstLine.substr(1); - } - - if (firstLine === `Literal '\\n' not allowed.`) { - // I won't localize this because none of the Ripgrep error messages are localized - return `Literal '\\n' currently not supported`; - } - - if (firstLine.startsWith('Literal ')) { - // Other unsupported chars - return firstLine; - } - - return undefined; -} - -export class RipgrepParser extends EventEmitter { - private remainder = ''; - private isDone = false; - private stringDecoder: NodeStringDecoder; - - private numResults = 0; - - constructor(private maxResults: number, private rootFolder: string, private previewOptions?: vscode.TextSearchPreviewOptions) { - super(); - this.stringDecoder = new StringDecoder(); - } - - public cancel(): void { - this.isDone = true; - } - - public flush(): void { - this.handleDecodedData(this.stringDecoder.end()); - } - - public handleData(data: Buffer | string): void { - const dataStr = typeof data === 'string' ? data : this.stringDecoder.write(data); - this.handleDecodedData(dataStr); - } - - private handleDecodedData(decodedData: string): void { - // If the previous data chunk didn't end in a newline, prepend it to this chunk - const dataStr = this.remainder ? - this.remainder + decodedData : - decodedData; - - const dataLines: string[] = dataStr.split(/\r\n|\n/); - this.remainder = dataLines[dataLines.length - 1] ? dataLines.pop() : ''; - - for (let l = 0; l < dataLines.length; l++) { - const line = dataLines[l]; - if (line) { // Empty line at the end of each chunk - this.handleLine(line); - } - } - } - - private handleLine(outputLine: string): void { - if (this.isDone) { - return; - } - - let parsedLine: any; - try { - parsedLine = JSON.parse(outputLine); - } catch (e) { - throw new Error(`malformed line from rg: ${outputLine}`); - } - - if (parsedLine.type === 'match') { - let hitLimit = false; - const uri = vscode.Uri.file(path.join(this.rootFolder, parsedLine.data.path.text)); - parsedLine.data.submatches.map((match: any) => { - if (hitLimit) { - return null; - } - - if (this.numResults >= this.maxResults) { - // Finish the line, then report the result below - hitLimit = true; - } - - return this.submatchToResult(parsedLine, match, uri); - }).forEach((result: any) => { - if (result) { - this.onResult(result); - } - }); - - if (hitLimit) { - this.cancel(); - this.emit('hitLimit'); - } - } - } - - private submatchToResult(parsedLine: any, match: any, uri: vscode.Uri): vscode.TextSearchResult { - const lineNumber = parsedLine.data.line_number - 1; - let matchText = parsedLine.data.lines.bytes ? - new Buffer(parsedLine.data.lines.bytes, 'base64').toString() : - parsedLine.data.lines.text; - let start = match.start; - let end = match.end; - if (lineNumber === 0) { - if (startsWithUTF8BOM(matchText)) { - matchText = stripUTF8BOM(matchText); - start -= 3; - end -= 3; - } - } - - const range = new vscode.Range(lineNumber, start, lineNumber, end); - return createTextSearchResult(uri, matchText, range, this.previewOptions); - } - - private onResult(match: vscode.TextSearchResult): void { - this.emit('result', match); - } -} - -function getRgArgs(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions): string[] { - const args = ['--hidden', '--heading', '--line-number', '--color', 'ansi', '--colors', 'path:none', '--colors', 'line:none', '--colors', 'match:fg:red', '--colors', 'match:style:nobold']; - args.push(query.isCaseSensitive ? '--case-sensitive' : '--ignore-case'); - - options.includes - .map(anchorGlob) - .forEach(globArg => args.push('-g', globArg)); - - options.excludes - .map(anchorGlob) - .forEach(rgGlob => args.push('-g', `!${rgGlob}`)); - - if (options.maxFileSize) { - args.push('--max-filesize', options.maxFileSize + ''); - } - - if (options.useIgnoreFiles) { - args.push('--no-ignore-parent'); - } else { - // Don't use .gitignore or .ignore - args.push('--no-ignore'); - } - - if (options.followSymlinks) { - args.push('--follow'); - } - - if (options.encoding) { - args.push('--encoding', options.encoding); - } - - // Ripgrep handles -- as a -- arg separator. Only --. - // - is ok, --- is ok, --some-flag is handled as query text. Need to special case. - if (query.pattern === '--') { - query.isRegExp = true; - query.pattern = '\\-\\-'; - } - - let searchPatternAfterDoubleDashes: Maybe; - if (query.isWordMatch) { - const regexp = createRegExp(query.pattern, !!query.isRegExp, { wholeWord: query.isWordMatch }); - const regexpStr = regexp.source.replace(/\\\//g, '/'); // RegExp.source arbitrarily returns escaped slashes. Search and destroy. - args.push('--regexp', regexpStr); - } else if (query.isRegExp) { - args.push('--regexp', fixRegexEndingPattern(query.pattern)); - } else { - searchPatternAfterDoubleDashes = query.pattern; - args.push('--fixed-strings'); - } - - args.push('--no-config'); - if (!options.useGlobalIgnoreFiles) { - args.push('--no-ignore-global'); - } - - args.push('--json'); - - // Folder to search - args.push('--'); - - if (searchPatternAfterDoubleDashes) { - // Put the query after --, in case the query starts with a dash - args.push(searchPatternAfterDoubleDashes); - } - - args.push('.'); - - return args; -} - -interface RegExpOptions { - matchCase?: boolean; - wholeWord?: boolean; - multiline?: boolean; - global?: boolean; -} - -function createRegExp(searchString: string, isRegex: boolean, options: RegExpOptions = {}): RegExp { - if (!searchString) { - throw new Error('Cannot create regex from empty string'); - } - if (!isRegex) { - searchString = escapeRegExpCharacters(searchString); - } - if (options.wholeWord) { - if (!/\B/.test(searchString.charAt(0))) { - searchString = '\\b' + searchString; - } - if (!/\B/.test(searchString.charAt(searchString.length - 1))) { - searchString = searchString + '\\b'; - } - } - let modifiers = ''; - if (options.global) { - modifiers += 'g'; - } - if (!options.matchCase) { - modifiers += 'i'; - } - if (options.multiline) { - modifiers += 'm'; - } - - return new RegExp(searchString, modifiers); -} - -/** - * Escapes regular expression characters in a given string - */ -function escapeRegExpCharacters(value: string): string { - return value.replace(/[\-\\\{\}\*\+\?\|\^\$\.\[\]\(\)\#]/g, '\\$&'); -} - -// -- UTF-8 BOM - -const UTF8_BOM = 65279; - -function startsWithUTF8BOM(str: string): boolean { - return !!(str && str.length > 0 && str.charCodeAt(0) === UTF8_BOM); -} - -function stripUTF8BOM(str: string): string { - return startsWithUTF8BOM(str) ? str.substr(1) : str; -} - -function fixRegexEndingPattern(pattern: string): string { - // Replace an unescaped $ at the end of the pattern with \r?$ - // Match $ preceeded by none or even number of literal \ - return pattern.match(/([^\\]|^)(\\\\)*\$$/) ? - pattern.replace(/\$$/, '\\r?$') : - pattern; -} \ No newline at end of file -- GitLab