提交 5359560e 编写于 作者: R Rob Lourens

Update ripgrep version, use new json output format

上级 1bf1bc2c
......@@ -15,7 +15,7 @@
"dependencies": {
"vscode-extension-telemetry": "0.0.22",
"vscode-nls": "^4.0.0",
"vscode-ripgrep": "1.1.0"
"vscode-ripgrep": "^1.2.0"
},
"devDependencies": {
"@types/node": "8.0.33",
......
......@@ -138,15 +138,6 @@ export function rgErrorMsgForDisplay(msg: string): Maybe<string> {
}
export class RipgrepParser extends EventEmitter {
private static readonly RESULT_REGEX = /^\u001b\[0m(\d+)\u001b\[0m:(.*)(\r?)/;
private static readonly FILE_REGEX = /^\u001b\[0m(.+)\u001b\[0m$/;
private static readonly ESC_CODE = '\u001b'.charCodeAt(0);
// public for test
public static readonly MATCH_START_MARKER = '\u001b[0m\u001b[31m';
public static readonly MATCH_END_MARKER = '\u001b[0m';
private currentFile = '';
private remainder = '';
private isDone = false;
private stringDecoder: NodeStringDecoder;
......@@ -181,108 +172,69 @@ export class RipgrepParser extends EventEmitter {
this.remainder = dataLines[dataLines.length - 1] ? <string>dataLines.pop() : '';
for (let l = 0; l < dataLines.length; l++) {
const outputLine = dataLines[l].trim();
if (this.isDone) {
break;
}
let r: Maybe<RegExpMatchArray>;
if (r = outputLine.match(RipgrepParser.RESULT_REGEX)) {
const lineNum = parseInt(r[1]) - 1;
let matchText = r[2];
// workaround https://github.com/BurntSushi/ripgrep/issues/416
// If the match line ended with \r, append a match end marker so the match isn't lost
if (r[3]) {
matchText += RipgrepParser.MATCH_END_MARKER;
}
// Line is a result - add to collected results for the current file path
this.handleMatchLine(lineNum, matchText);
} else if (r = outputLine.match(RipgrepParser.FILE_REGEX)) {
this.currentFile = r[1];
} else {
// Line is empty (or malformed)
const line = dataLines[l];
if (line) { // Empty line at the end of each chunk
this.handleLine(line);
}
}
}
private handleMatchLine(lineNum: number, lineText: string): void {
if (lineNum === 0) {
lineText = stripUTF8BOM(lineText);
private handleLine(outputLine: string): void {
if (this.isDone) {
return;
}
let lastMatchEndPos = 0;
let matchTextStartPos = -1;
// Track positions with color codes subtracted - offsets in the final text preview result
let matchTextStartRealIdx = -1;
let textRealIdx = 0;
let hitLimit = false;
const realTextParts: string[] = [];
const lineMatches: vscode.Range[] = [];
for (let i = 0; i < lineText.length - (RipgrepParser.MATCH_END_MARKER.length - 1);) {
if (lineText.charCodeAt(i) === RipgrepParser.ESC_CODE) {
if (lineText.substr(i, RipgrepParser.MATCH_START_MARKER.length) === RipgrepParser.MATCH_START_MARKER) {
// Match start
const chunk = lineText.slice(lastMatchEndPos, i);
realTextParts.push(chunk);
i += RipgrepParser.MATCH_START_MARKER.length;
matchTextStartPos = i;
matchTextStartRealIdx = textRealIdx;
} else if (lineText.substr(i, RipgrepParser.MATCH_END_MARKER.length) === RipgrepParser.MATCH_END_MARKER) {
// Match end
const chunk = lineText.slice(matchTextStartPos, i);
realTextParts.push(chunk);
if (!hitLimit) {
const startCol = matchTextStartRealIdx;
const endCol = textRealIdx;
// actually have to finish parsing the line, and use the real ones
lineMatches.push(new vscode.Range(lineNum, startCol, lineNum, endCol));
}
matchTextStartPos = -1;
matchTextStartRealIdx = -1;
i += RipgrepParser.MATCH_END_MARKER.length;
lastMatchEndPos = i;
this.numResults++;
let parsedLine: any;
try {
parsedLine = JSON.parse(outputLine);
} catch (e) {
throw new Error(`malformed line from rg: ${outputLine}`);
}
// Check hit maxResults limit
if (this.numResults >= this.maxResults) {
// Finish the line, then report the result below
hitLimit = true;
}
} else {
// ESC char in file
i++;
textRealIdx++;
if (parsedLine.type === 'match') {
let hitLimit = false;
const uri = vscode.Uri.file(path.join(this.rootFolder, parsedLine.data.path.text));
parsedLine.data.submatches.map((match: any) => {
if (hitLimit) {
return null;
}
} else {
// Some other char
i++;
textRealIdx++;
}
}
const chunk = lineText.slice(lastMatchEndPos);
realTextParts.push(chunk);
if (this.numResults >= this.maxResults) {
// Finish the line, then report the result below
hitLimit = true;
}
// Get full real text line without color codes
const previewText = realTextParts.join('');
return this.submatchToResult(parsedLine, match, uri);
}).forEach((result: any) => {
if (result) {
this.onResult(result);
}
});
const uri = vscode.Uri.file(path.join(this.rootFolder, this.currentFile));
lineMatches
.map(range => createTextSearchResult(uri, previewText, range, this.previewOptions))
.forEach(match => this.onResult(match));
if (hitLimit) {
this.cancel();
this.emit('hitLimit');
}
}
}
if (hitLimit) {
this.cancel();
this.emit('hitLimit');
private submatchToResult(parsedLine: any, match: any, uri: vscode.Uri): vscode.TextSearchResult {
const lineNumber = parsedLine.data.line_number - 1;
let matchText = parsedLine.data.lines.bytes ?
new Buffer(parsedLine.data.lines.bytes, 'base64').toString() :
parsedLine.data.lines.text;
let start = match.start;
let end = match.end;
if (lineNumber === 0) {
if (startsWithUTF8BOM(matchText)) {
matchText = stripUTF8BOM(matchText);
start -= 3;
end -= 3;
}
}
const range = new vscode.Range(lineNumber, start, lineNumber, end);
return createTextSearchResult(uri, matchText, range, this.previewOptions);
}
private onResult(match: vscode.TextSearchResult): void {
......@@ -345,6 +297,8 @@ function getRgArgs(query: vscode.TextSearchQuery, options: vscode.TextSearchOpti
args.push('--no-ignore-global');
}
args.push('--json');
// Folder to search
args.push('--');
......
......@@ -1814,10 +1814,10 @@ vscode-nls@^4.0.0:
resolved "https://registry.yarnpkg.com/vscode-nls/-/vscode-nls-4.0.0.tgz#4001c8a6caba5cedb23a9c5ce1090395c0e44002"
integrity sha512-qCfdzcH+0LgQnBpZA53bA32kzp9rpq/f66Som577ObeuDlFIrtbEJ+A/+CCxjIh4G8dpJYNCKIsxpRAHIfsbNw==
vscode-ripgrep@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.1.0.tgz#93c1e39d88342ee1b15530a12898ce930d511948"
integrity sha512-7Bsa13vk1mtjg1PfkjDwDloy2quDxnhvCjRwVMaYwFcwzgIGkai5TuNuziWisqUeKbSnFQsoIylSaqb+sIpFXA==
vscode-ripgrep@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.2.0.tgz#f9daef7332c968a1044afc42a31788ac49e315af"
integrity sha512-OZAH4+euvPqf1p8bsl6q+l3XMXit2Ef8B8j2bQlAGJpeJ//2JdAH8Ih0kHNPcPhvnp+M3w+ISAXinfoykwFc7A==
vscode@^1.1.17:
version "1.1.17"
......
......@@ -235,8 +235,8 @@ export class FileWalker {
// Mac: uses NFD unicode form on disk, but we want NFC
const normalized = leftover + (isMac ? normalization.normalizeNFC(stdout) : stdout);
const relativeFiles = normalized.split(useRipgrep ? '\n' : '\n./');
if (!useRipgrep && first && normalized.length >= 2) {
const relativeFiles = normalized.split('\n./');
if (first && normalized.length >= 2) {
first = false;
relativeFiles[0] = relativeFiles[0].trim().substr(2);
}
......
......@@ -12,10 +12,11 @@ import * as objects from 'vs/base/common/objects';
import * as paths from 'vs/base/common/paths';
import * as platform from 'vs/base/common/platform';
import * as strings from 'vs/base/common/strings';
import { URI } from 'vs/base/common/uri';
import { TPromise } from 'vs/base/common/winjs.base';
import * as encoding from 'vs/base/node/encoding';
import * as extfs from 'vs/base/node/extfs';
import { IRange, Range } from 'vs/editor/common/core/range';
import { Range } from 'vs/editor/common/core/range';
import { IProgress, ITextSearchPreviewOptions, ITextSearchStats, TextSearchResult } from 'vs/platform/search/common/search';
import { rgPath } from 'vscode-ripgrep';
import { FileMatch, IFolderSearch, IRawSearch, ISerializedFileMatch, ISerializedSearchSuccess } from './search';
......@@ -77,7 +78,7 @@ export class RipgrepEngine {
this.rgProc = cp.spawn(rgDiskPath, rgArgs.args, { cwd });
process.once('exit', this.killRgProcFn);
this.ripgrepParser = new RipgrepParser(this.config.maxResults, cwd, this.config.extraFiles, this.config.previewOptions);
this.ripgrepParser = new RipgrepParser(this.config.maxResults, cwd, this.config.previewOptions);
this.ripgrepParser.on('result', (match: ISerializedFileMatch) => {
if (this.postProcessExclusions) {
const handleResultP = (<TPromise<string>>this.postProcessExclusions(match.path, undefined, glob.hasSiblingPromiseFn(() => getSiblings(match.path))))
......@@ -183,25 +184,16 @@ export function rgErrorMsgForDisplay(msg: string): string | undefined {
}
export class RipgrepParser extends EventEmitter {
private static readonly RESULT_REGEX = /^\u001b\[0m(\d+)\u001b\[0m:(.*)(\r?)/;
private static readonly FILE_REGEX = /^\u001b\[0m(.+)\u001b\[0m$/;
public static readonly MATCH_START_MARKER = '\u001b[0m\u001b[31m';
public static readonly MATCH_END_MARKER = '\u001b[0m';
private fileMatch: FileMatch;
private remainder: string;
private isDone: boolean;
private stringDecoder: NodeStringDecoder;
private extraSearchFiles: string[];
private numResults = 0;
constructor(private maxResults: number, private rootFolder: string, extraFiles?: string[], private previewOptions?: ITextSearchPreviewOptions) {
constructor(private maxResults: number, private rootFolder: string, private previewOptions?: ITextSearchPreviewOptions) {
super();
this.stringDecoder = new StringDecoder();
this.extraSearchFiles = extraFiles || [];
}
public cancel(): void {
......@@ -232,111 +224,57 @@ export class RipgrepParser extends EventEmitter {
for (let l = 0; l < dataLines.length; l++) {
const outputLine = dataLines[l].trim();
if (this.isDone) {
break;
}
let r: RegExpMatchArray;
if (r = outputLine.match(RipgrepParser.RESULT_REGEX)) {
const lineNum = parseInt(r[1]) - 1;
let matchText = r[2];
// workaround https://github.com/BurntSushi/ripgrep/issues/416
// If the match line ended with \r, append a match end marker so the match isn't lost
if (r[3]) {
matchText += RipgrepParser.MATCH_END_MARKER;
}
// Line is a result - add to collected results for the current file path
this.handleMatchLine(outputLine, lineNum, matchText);
} else if (r = outputLine.match(RipgrepParser.FILE_REGEX)) {
// Line is a file path - send all collected results for the previous file path
if (this.fileMatch) {
this.onResult();
}
this.fileMatch = this.getFileMatch(r[1]);
} else {
// Line is empty (or malformed)
if (outputLine) {
this.handleLine(outputLine);
}
}
}
private getFileMatch(relativeOrAbsolutePath: string): FileMatch {
const absPath = path.isAbsolute(relativeOrAbsolutePath) ?
relativeOrAbsolutePath :
path.join(this.rootFolder, relativeOrAbsolutePath);
return new FileMatch(absPath);
}
private handleLine(outputLine: string): void {
if (this.isDone) {
return;
}
private handleMatchLine(outputLine: string, lineNum: number, text: string): void {
if (lineNum === 0) {
text = strings.stripUTF8BOM(text);
let parsedLine: any;
try {
parsedLine = JSON.parse(outputLine);
} catch (e) {
throw new Error(`malformed line from rg: ${outputLine}`);
}
if (!this.fileMatch) {
// When searching a single file and no folderQueries, rg does not print the file line, so create it here
const singleFile = this.extraSearchFiles[0];
if (!singleFile) {
throw new Error('Got match line for unknown file');
if (parsedLine.type === 'begin') {
const path = bytesOrTextToString(parsedLine.data.path);
this.fileMatch = this.getFileMatch(path);
} else if (parsedLine.type === 'match') {
this.handleMatchLine(parsedLine);
} else if (parsedLine.type === 'end') {
if (this.fileMatch) {
this.onResult();
}
this.fileMatch = this.getFileMatch(singleFile);
}
}
let lastMatchEndPos = 0;
let matchTextStartPos = -1;
// Track positions with color codes subtracted - offsets in the final text preview result
let matchTextStartRealIdx = -1;
let textRealIdx = 0;
private handleMatchLine(parsedLine: any): void {
let hitLimit = false;
const matchRanges: IRange[] = [];
const realTextParts: string[] = [];
for (let i = 0; i < text.length - (RipgrepParser.MATCH_END_MARKER.length - 1);) {
if (text.substr(i, RipgrepParser.MATCH_START_MARKER.length) === RipgrepParser.MATCH_START_MARKER) {
// Match start
const chunk = text.slice(lastMatchEndPos, i);
realTextParts.push(chunk);
i += RipgrepParser.MATCH_START_MARKER.length;
matchTextStartPos = i;
matchTextStartRealIdx = textRealIdx;
} else if (text.substr(i, RipgrepParser.MATCH_END_MARKER.length) === RipgrepParser.MATCH_END_MARKER) {
// Match end
const chunk = text.slice(matchTextStartPos, i);
realTextParts.push(chunk);
if (!hitLimit) {
matchRanges.push(new Range(lineNum, matchTextStartRealIdx, lineNum, textRealIdx));
}
matchTextStartPos = -1;
matchTextStartRealIdx = -1;
i += RipgrepParser.MATCH_END_MARKER.length;
lastMatchEndPos = i;
this.numResults++;
// Check hit maxResults limit
if (this.numResults >= this.maxResults) {
// Finish the line, then report the result below
hitLimit = true;
}
} else {
i++;
textRealIdx++;
const uri = URI.file(path.join(this.rootFolder, parsedLine.data.path.text));
parsedLine.data.submatches.map((match: any) => {
if (hitLimit) {
return null;
}
}
const chunk = text.slice(lastMatchEndPos);
realTextParts.push(chunk);
this.numResults++;
if (this.numResults >= this.maxResults) {
// Finish the line, then report the result below
hitLimit = true;
}
// Replace preview with version without color codes
const preview = realTextParts.join('');
matchRanges
.map(r => new TextSearchResult(preview, r, this.previewOptions))
.forEach(m => this.fileMatch.addMatch(m));
return this.submatchToResult(parsedLine, match, uri);
}).forEach((result: any) => {
if (result) {
this.fileMatch.addMatch(result);
}
});
if (hitLimit) {
this.cancel();
......@@ -345,12 +283,43 @@ export class RipgrepParser extends EventEmitter {
}
}
private submatchToResult(parsedLine: any, match: any, uri: URI): TextSearchResult {
const lineNumber = parsedLine.data.line_number - 1;
let matchText = bytesOrTextToString(parsedLine.data.lines);
let start = match.start;
let end = match.end;
if (lineNumber === 0) {
if (strings.startsWithUTF8BOM(matchText)) {
matchText = strings.stripUTF8BOM(matchText);
start -= 3;
end -= 3;
}
}
const range = new Range(lineNumber, start, lineNumber, end);
return new TextSearchResult(matchText, range, this.previewOptions);
}
private getFileMatch(relativeOrAbsolutePath: string): FileMatch {
const absPath = path.isAbsolute(relativeOrAbsolutePath) ?
relativeOrAbsolutePath :
path.join(this.rootFolder, relativeOrAbsolutePath);
return new FileMatch(absPath);
}
private onResult(): void {
this.emit('result', this.fileMatch.serialize());
this.fileMatch = null;
}
}
function bytesOrTextToString(obj: any): string {
return obj.bytes ?
new Buffer(obj.bytes, 'base64').toString() :
obj.text;
}
export interface IRgGlobResult {
globArgs: string[];
siblingClauses: glob.IExpression;
......@@ -519,6 +488,8 @@ function getRgArgs(config: IRawSearch) {
args.push('--no-ignore-global');
}
args.push('--json');
// Folder to search
args.push('--');
......
......@@ -4,195 +4,8 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import * as path from 'path';
import * as arrays from 'vs/base/common/arrays';
import * as platform from 'vs/base/common/platform';
import { fixDriveC, fixRegexEndingPattern, getAbsoluteGlob, RipgrepParser } from 'vs/workbench/services/search/node/ripgrepTextSearch';
import { ISerializedFileMatch } from 'vs/workbench/services/search/node/search';
suite('RipgrepParser', () => {
const rootFolder = '/workspace';
const fileSectionEnd = '\n';
function getFileLine(relativePath: string): string {
return `\u001b\[0m${relativePath}\u001b\[0m`;
}
function getMatchLine(lineNum: number, matchParts: string[]): string {
let matchLine = `\u001b\[0m${lineNum}\u001b\[0m:` +
`${matchParts.shift()}${RipgrepParser.MATCH_START_MARKER}${matchParts.shift()}${RipgrepParser.MATCH_END_MARKER}${matchParts.shift()}`;
while (matchParts.length) {
matchLine += `${RipgrepParser.MATCH_START_MARKER}${matchParts.shift()}${RipgrepParser.MATCH_END_MARKER}${matchParts.shift() || ''}`;
}
return matchLine;
}
function parseInputStrings(inputChunks: string[]): ISerializedFileMatch[] {
return parseInput(inputChunks.map(chunk => Buffer.from(chunk)));
}
function parseInput(inputChunks: Buffer[]): ISerializedFileMatch[] {
const matches: ISerializedFileMatch[] = [];
const rgp = new RipgrepParser(1e6, rootFolder);
rgp.on('result', (match: ISerializedFileMatch) => {
matches.push(match);
});
inputChunks.forEach(chunk => rgp.handleData(chunk));
rgp.flush();
return matches;
}
function halve(str: string) {
const halfIdx = Math.floor(str.length / 2);
return [str.substr(0, halfIdx), str.substr(halfIdx)];
}
function arrayOfChars(str: string) {
const chars = [];
for (let char of str) {
chars.push(char);
}
return chars;
}
test('Parses one chunk', () => {
const input = [
[getFileLine('a.txt'), getMatchLine(1, ['before', 'match', 'after']), getMatchLine(2, ['before', 'match', 'after']), fileSectionEnd].join('\n')
];
const results = parseInputStrings(input);
assert.equal(results.length, 1);
assert.deepEqual(results[0],
<ISerializedFileMatch>{
numMatches: 2,
path: path.join(rootFolder, 'a.txt'),
matches: [
{
preview: {
match: {
endColumn: 11,
endLineNumber: 0,
startColumn: 6,
startLineNumber: 0,
},
text: 'beforematchafter'
},
range: {
endColumn: 11,
endLineNumber: 0,
startColumn: 6,
startLineNumber: 0,
}
},
{
preview: {
match: {
endColumn: 11,
endLineNumber: 0,
startColumn: 6,
startLineNumber: 0,
},
text: 'beforematchafter'
},
range: {
endColumn: 11,
endLineNumber: 1,
startColumn: 6,
startLineNumber: 1,
}
}
]
});
});
test('Parses multiple chunks broken at file sections', () => {
const input = [
[getFileLine('a.txt'), getMatchLine(1, ['before', 'match', 'after']), getMatchLine(2, ['before', 'match', 'after']), fileSectionEnd].join('\n'),
[getFileLine('b.txt'), getMatchLine(1, ['before', 'match', 'after']), getMatchLine(2, ['before', 'match', 'after']), fileSectionEnd].join('\n'),
[getFileLine('c.txt'), getMatchLine(1, ['before', 'match', 'after']), getMatchLine(2, ['before', 'match', 'after']), fileSectionEnd].join('\n')
];
const results = parseInputStrings(input);
assert.equal(results.length, 3);
results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
});
const singleLineChunks = [
getFileLine('a.txt'),
getMatchLine(1, ['before', 'match', 'after']),
getMatchLine(2, ['before', 'match', 'after']),
fileSectionEnd,
getFileLine('b.txt'),
getMatchLine(1, ['before', 'match', 'after']),
getMatchLine(2, ['before', 'match', 'after']),
fileSectionEnd,
getFileLine('c.txt'),
getMatchLine(1, ['before', 'match', 'after']),
getMatchLine(2, ['before', 'match', 'after']),
fileSectionEnd
];
test('Parses multiple chunks broken at each line', () => {
const input = singleLineChunks.map(chunk => chunk + '\n');
const results = parseInputStrings(input);
assert.equal(results.length, 3);
results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
});
test('Parses multiple chunks broken in the middle of each line', () => {
const input = arrays.flatten(singleLineChunks
.map(chunk => chunk + '\n')
.map(halve));
const results = parseInputStrings(input);
assert.equal(results.length, 3);
results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
});
test('Parses multiple chunks broken at each character', () => {
const input = arrays.flatten(singleLineChunks
.map(chunk => chunk + '\n')
.map(arrayOfChars));
const results = parseInputStrings(input);
assert.equal(results.length, 3);
results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
});
test('Parses chunks broken before newline', () => {
const input = singleLineChunks
.map(chunk => '\n' + chunk);
const results = parseInputStrings(input);
assert.equal(results.length, 3);
results.forEach(fileResult => assert.equal(fileResult.numMatches, 2));
});
test('Parses chunks broken in the middle of a multibyte character', () => {
const text = getFileLine('foo/bar') + '\n' + getMatchLine(0, ['before漢', 'match', 'after']) + '\n';
const buf = Buffer.from(text);
// Split the buffer at every possible position - it should still be parsed correctly
for (let i = 0; i < buf.length; i++) {
const inputBufs = [
buf.slice(0, i),
buf.slice(i)
];
const results = parseInput(inputBufs);
assert.equal(results.length, 1);
assert.equal(results[0].matches.length, 1);
assert.equal(results[0].matches[0].range.startColumn, 7);
assert.equal(results[0].matches[0].range.endColumn, 12);
}
});
});
import { fixDriveC, fixRegexEndingPattern, getAbsoluteGlob } from 'vs/workbench/services/search/node/ripgrepTextSearch';
suite('RipgrepTextSearch - etc', () => {
function testGetAbsGlob(params: string[]): void {
......
......@@ -9339,10 +9339,10 @@ vscode-nsfw@1.0.17:
nan "^2.0.0"
promisify-node "^0.3.0"
vscode-ripgrep@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.1.0.tgz#93c1e39d88342ee1b15530a12898ce930d511948"
integrity sha512-7Bsa13vk1mtjg1PfkjDwDloy2quDxnhvCjRwVMaYwFcwzgIGkai5TuNuziWisqUeKbSnFQsoIylSaqb+sIpFXA==
vscode-ripgrep@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.2.0.tgz#f9daef7332c968a1044afc42a31788ac49e315af"
integrity sha512-OZAH4+euvPqf1p8bsl6q+l3XMXit2Ef8B8j2bQlAGJpeJ//2JdAH8Ih0kHNPcPhvnp+M3w+ISAXinfoykwFc7A==
vscode-textmate@^4.0.1:
version "4.0.1"
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册