Fix text search around multibyte characters

src/vs/workbench/services/search/node/ripgrepTextSearchEngine.ts

@@ -222,9 +222,13 @@ export class RipgrepParser extends EventEmitter {
 		let matchText = bytesOrTextToString(match.match);
 		const newlineMatches = matchText.match(/\n/g);
 		const newlines = newlineMatches ? newlineMatches.length : 0;
-		let startCol = match.start;
+
+		const textBytes = new Buffer(lineText);
+		let startCol = textBytes.slice(0, match.start).toString().length;
+		const endChars = startCol + textBytes.slice(match.start, match.end).toString().length;
+
 		const endLineNumber = lineNumber + newlines;
-		let endCol = match.end - (lineText.lastIndexOf('\n', lineText.length - 2) + 1);
+		let endCol = endChars - (lineText.lastIndexOf('\n', lineText.length - 2) + 1);
 
 		if (lineNumber === 0) {
 			if (startsWithUTF8BOM(matchText)) {

src/vs/workbench/services/search/test/node/textSearch.integrationTest.ts

@@ -49,22 +49,25 @@ function doLegacySearchTest(config: ITextQuery, expectedResultCount: number | Fu
 	});
 }
 
-function doRipgrepSearchTest(query: ITextQuery, expectedResultCount: number | Function): TPromise<void> {
+function doRipgrepSearchTest(query: ITextQuery, expectedResultCount: number | Function): TPromise<ISerializedFileMatch[]> {
 	let engine = new TextSearchEngineAdapter(query);
 
 	let c = 0;
-	return engine.search(new CancellationTokenSource().token, (results) => {
-		if (results) {
-			c += results.reduce((acc, cur) => acc + cur.numMatches, 0);
+	const results: ISerializedFileMatch[] = [];
+	return engine.search(new CancellationTokenSource().token, _results => {
+		if (_results) {
+			c += _results.reduce((acc, cur) => acc + cur.numMatches, 0);
+			results.push(..._results);
 		}
-	}, () => { }).then(
-		() => {
-			if (typeof expectedResultCount === 'function') {
-				assert(expectedResultCount(c));
-			} else {
-				assert.equal(c, expectedResultCount, `rg ${c} !== ${expectedResultCount}`);
-			}
-		});
+	}, () => { }).then(() => {
+		if (typeof expectedResultCount === 'function') {
+			assert(expectedResultCount(c));
+		} else {
+			assert.equal(c, expectedResultCount, `rg ${c} !== ${expectedResultCount}`);
+		}
+
+		return results;
+	});
 }
 
 function doSearchTest(query: ITextQuery, expectedResultCount: number) {
@@ -373,6 +376,24 @@ suite('Search-integration', function () {
 				assert.equal(err.message, 'The literal \'"\\n"\' is not allowed in a regex');
 			});
 		});
+
+		test('Text: 语', () => {
+			const config = <ITextQuery>{
+				type: QueryType.Text,
+				folderQueries: ROOT_FOLDER_QUERY,
+				contentPattern: { pattern: '语' }
+			};
+
+			return doRipgrepSearchTest(config, 1).then(results => {
+				const matchRange = results[0].matches[0].range;
+				assert.deepEqual(matchRange, {
+					startLineNumber: 0,
+					startColumn: 1,
+					endLineNumber: 0,
+					endColumn: 2
+				});
+			});
+		});
 	});
 });