Merge pull request #43349 from Microsoft/fix-40941

Fix #40941 - Suggest trigger at language boundaries

src/vs/editor/common/model/textModel.ts

@@ -1764,8 +1764,39 @@ export class TextModel extends Disposable implements model.ITextModel {
 		const position = this.validatePosition(_position);
 		const lineContent = this.getLineContent(position.lineNumber);
 		const lineTokens = this._getLineTokens(position.lineNumber);
-		const offset = position.column - 1;
-		const tokenIndex = lineTokens.findTokenIndexAtOffset(offset);
+		const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
+
+		// (1). First try checking right biased word
+		const [rbStartOffset, rbEndOffset] = TextModel._findLanguageBoundaries(lineTokens, tokenIndex);
+		const rightBiasedWord = getWordAtText(
+			position.column,
+			LanguageConfigurationRegistry.getWordDefinition(lineTokens.getLanguageId(tokenIndex)),
+			lineContent.substring(rbStartOffset, rbEndOffset),
+			rbStartOffset
+		);
+		if (rightBiasedWord) {
+			return rightBiasedWord;
+		}
+
+		// (2). Else, if we were at a language boundary, check the left biased word
+		if (tokenIndex > 0 && rbStartOffset === position.column - 1) {
+			// edge case, where `position` sits between two tokens belonging to two different languages
+			const [lbStartOffset, lbEndOffset] = TextModel._findLanguageBoundaries(lineTokens, tokenIndex - 1);
+			const leftBiasedWord = getWordAtText(
+				position.column,
+				LanguageConfigurationRegistry.getWordDefinition(lineTokens.getLanguageId(tokenIndex - 1)),
+				lineContent.substring(lbStartOffset, lbEndOffset),
+				lbStartOffset
+			);
+			if (leftBiasedWord) {
+				return leftBiasedWord;
+			}
+		}
+
+		return null;
+	}
+
+	private static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] {
 		const languageId = lineTokens.getLanguageId(tokenIndex);
 
 		// go left until a different language is hit
@@ -1780,12 +1811,7 @@ export class TextModel extends Disposable implements model.ITextModel {
 			endOffset = lineTokens.getEndOffset(i);
 		}
 
-		return getWordAtText(
-			position.column,
-			LanguageConfigurationRegistry.getWordDefinition(languageId),
-			lineContent.substring(startOffset, endOffset),
-			startOffset
-		);
+		return [startOffset, endOffset];
 	}
 
 	public getWordUntilPosition(position: IPosition): model.IWordAtPosition {

src/vs/editor/contrib/suggest/suggestModel.ts

@@ -47,6 +47,7 @@ export class LineContext {
 		}
 		const pos = editor.getPosition();
 		model.tokenizeIfCheap(pos.lineNumber);
+
 		const word = model.getWordAtPosition(pos);
 		if (!word) {
 			return false;

src/vs/editor/contrib/suggest/test/suggestModel.test.ts

@@ -7,10 +7,10 @@
 import * as assert from 'assert';
 import { Event } from 'vs/base/common/event';
 import URI from 'vs/base/common/uri';
-import { IDisposable, dispose } from 'vs/base/common/lifecycle';
+import { IDisposable, dispose, Disposable } from 'vs/base/common/lifecycle';
 import { TextModel } from 'vs/editor/common/model/textModel';
 import { Handler } from 'vs/editor/common/editorCommon';
-import { ISuggestSupport, ISuggestResult, SuggestRegistry, SuggestTriggerKind } from 'vs/editor/common/modes';
+import { ISuggestSupport, ISuggestResult, SuggestRegistry, SuggestTriggerKind, LanguageIdentifier, TokenizationRegistry, IState, MetadataConsts } from 'vs/editor/common/modes';
 import { SuggestModel, LineContext } from 'vs/editor/contrib/suggest/suggestModel';
 import { TestCodeEditor, MockScopeLocation } from 'vs/editor/test/browser/testCodeEditor';
 import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection';
@@ -27,6 +27,10 @@ import { SuggestController } from 'vs/editor/contrib/suggest/suggestController';
 import { IStorageService, NullStorageService } from 'vs/platform/storage/common/storage';
 import { SnippetController2 } from 'vs/editor/contrib/snippet/snippetController2';
 import { ISelectedSuggestion } from 'vs/editor/contrib/suggest/suggestWidget';
+import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
+import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
+import { TokenizationResult2 } from 'vs/editor/common/core/token';
+import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
 
 function createMockEditor(model: TextModel): TestCodeEditor {
 	const contextKeyService = new MockContextKeyService();
@@ -43,33 +47,91 @@ function createMockEditor(model: TextModel): TestCodeEditor {
 }
 
 suite('SuggestModel - Context', function () {
+	const OUTER_LANGUAGE_ID = new LanguageIdentifier('outerMode', 3);
+	const INNER_LANGUAGE_ID = new LanguageIdentifier('innerMode', 4);
+
+	class OuterMode extends MockMode {
+		constructor() {
+			super(OUTER_LANGUAGE_ID);
+			this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));
+
+			this._register(TokenizationRegistry.register(this.getLanguageIdentifier().language, {
+				getInitialState: (): IState => NULL_STATE,
+				tokenize: undefined,
+				tokenize2: (line: string, state: IState): TokenizationResult2 => {
+					const tokensArr: number[] = [];
+					let prevLanguageId: LanguageIdentifier = undefined;
+					for (let i = 0; i < line.length; i++) {
+						const languageId = (line.charAt(i) === 'x' ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);
+						if (prevLanguageId !== languageId) {
+							tokensArr.push(i);
+							tokensArr.push((languageId.id << MetadataConsts.LANGUAGEID_OFFSET));
+						}
+						prevLanguageId = languageId;
+					}
+
+					const tokens = new Uint32Array(tokensArr.length);
+					for (let i = 0; i < tokens.length; i++) {
+						tokens[i] = tokensArr[i];
+					}
+					return new TokenizationResult2(tokens, state);
+				}
+			}));
+		}
+	}
 
-	let model: TextModel;
+	class InnerMode extends MockMode {
+		constructor() {
+			super(INNER_LANGUAGE_ID);
+			this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));
+		}
+	}
 
-	setup(function () {
-		model = TextModel.createFromString('Das Pferd frisst keinen Gurkensalat - Philipp Reis 1861.\nWer hat\'s erfunden?');
+	const assertAutoTrigger = (model: TextModel, offset: number, expected: boolean, message?: string): void => {
+		const pos = model.getPositionAt(offset);
+		const editor = createMockEditor(model);
+		editor.setPosition(pos);
+		assert.equal(LineContext.shouldAutoTrigger(editor), expected, message);
+		editor.dispose();
+	};
+
+	let disposables: Disposable[] = [];
+
+	setup(() => {
+		disposables = [];
 	});
 
 	teardown(function () {
-		model.dispose();
+		dispose(disposables);
+		disposables = [];
 	});
 
 	test('Context - shouldAutoTrigger', function () {
+		const model = TextModel.createFromString('Das Pferd frisst keinen Gurkensalat - Philipp Reis 1861.\nWer hat\'s erfunden?');
+		disposables.push(model);
 
-		function assertAutoTrigger(offset: number, expected: boolean): void {
-			const pos = model.getPositionAt(offset);
-			const editor = createMockEditor(model);
-			editor.setPosition(pos);
-			assert.equal(LineContext.shouldAutoTrigger(editor), expected);
-			editor.dispose();
-		}
-
-		assertAutoTrigger(3, true); // end of word, Das|
-		assertAutoTrigger(4, false); // no word Das |
-		assertAutoTrigger(1, false); // middle of word D|as
-		assertAutoTrigger(55, false); // number, 1861|
+		assertAutoTrigger(model, 3, true, 'end of word, Das|');
+		assertAutoTrigger(model, 4, false, 'no word Das |');
+		assertAutoTrigger(model, 1, false, 'middle of word D|as');
+		assertAutoTrigger(model, 55, false, 'number, 1861|');
 	});
 
+	test('shouldAutoTrigger at embedded language boundaries', () => {
+		const outerMode = new OuterMode();
+		const innerMode = new InnerMode();
+		disposables.push(outerMode, innerMode);
+
+		const model = TextModel.createFromString('a<xx>a<x>', undefined, outerMode.getLanguageIdentifier());
+		disposables.push(model);
+
+		assertAutoTrigger(model, 1, true, 'a|<x — should trigger at end of word');
+		assertAutoTrigger(model, 2, false, 'a<|x — should NOT trigger at start of word');
+		assertAutoTrigger(model, 3, false, 'a<x|x —  should NOT trigger in middle of word');
+		assertAutoTrigger(model, 4, true, 'a<xx|> — should trigger at boundary between languages');
+		assertAutoTrigger(model, 5, false, 'a<xx>|a — should NOT trigger at start of word');
+		assertAutoTrigger(model, 6, true, 'a<xx>a|< — should trigger at end of word');
+		assertAutoTrigger(model, 8, true, 'a<xx>a<x|> — should trigger at end of word at boundary');
+	});
 });
 
 suite('SuggestModel - TriggerAndCancelOracle', function () {

src/vs/editor/test/common/model/model.test.ts

@@ -13,6 +13,12 @@ import {
 	ModelRawLinesDeleted, ModelRawLinesInserted
 } from 'vs/editor/common/model/textModelEvents';
 import { TextModel } from 'vs/editor/common/model/textModel';
+import { LanguageIdentifier, TokenizationRegistry, IState, MetadataConsts } from 'vs/editor/common/modes';
+import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
+import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
+import { TokenizationResult2 } from 'vs/editor/common/core/token';
+import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
+import { dispose, Disposable } from 'vs/base/common/lifecycle';
 
 // --------- utils
 
@@ -365,18 +371,62 @@ suite('Editor Model - Model Line Separators', () => {
 
 suite('Editor Model - Words', () => {
 
-	var thisModel: TextModel;
+	const OUTER_LANGUAGE_ID = new LanguageIdentifier('outerMode', 3);
+	const INNER_LANGUAGE_ID = new LanguageIdentifier('innerMode', 4);
+
+	class OuterMode extends MockMode {
+		constructor() {
+			super(OUTER_LANGUAGE_ID);
+			this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));
+
+			this._register(TokenizationRegistry.register(this.getLanguageIdentifier().language, {
+				getInitialState: (): IState => NULL_STATE,
+				tokenize: undefined,
+				tokenize2: (line: string, state: IState): TokenizationResult2 => {
+					const tokensArr: number[] = [];
+					let prevLanguageId: LanguageIdentifier = undefined;
+					for (let i = 0; i < line.length; i++) {
+						const languageId = (line.charAt(i) === 'x' ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);
+						if (prevLanguageId !== languageId) {
+							tokensArr.push(i);
+							tokensArr.push((languageId.id << MetadataConsts.LANGUAGEID_OFFSET));
+						}
+						prevLanguageId = languageId;
+					}
+
+					const tokens = new Uint32Array(tokensArr.length);
+					for (let i = 0; i < tokens.length; i++) {
+						tokens[i] = tokensArr[i];
+					}
+					return new TokenizationResult2(tokens, state);
+				}
+			}));
+		}
+	}
+
+	class InnerMode extends MockMode {
+		constructor() {
+			super(INNER_LANGUAGE_ID);
+			this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));
+		}
+	}
+
+	let disposables: Disposable[] = [];
 
 	setup(() => {
-		var text = ['This text has some  words. '];
-		thisModel = TextModel.createFromString(text.join('\n'));
+		disposables = [];
 	});
 
 	teardown(() => {
-		thisModel.dispose();
+		dispose(disposables);
+		disposables = [];
 	});
 
 	test('Get word at position', () => {
+		const text = ['This text has some  words. '];
+		const thisModel = TextModel.createFromString(text.join('\n'));
+		disposables.push(thisModel);
+
 		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 1)), { word: 'This', startColumn: 1, endColumn: 5 });
 		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 2)), { word: 'This', startColumn: 1, endColumn: 5 });
 		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 4)), { word: 'This', startColumn: 1, endColumn: 5 });
@@ -389,4 +439,21 @@ suite('Editor Model - Words', () => {
 		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 27)), null);
 		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 28)), null);
 	});
+
+	test('getWordAtPosition at embedded language boundaries', () => {
+		const outerMode = new OuterMode();
+		const innerMode = new InnerMode();
+		disposables.push(outerMode, innerMode);
+
+		const model = TextModel.createFromString('ab<xx>ab<x>', undefined, outerMode.getLanguageIdentifier());
+		disposables.push(model);
+
+		assert.deepEqual(model.getWordAtPosition(new Position(1, 1)), { word: 'ab', startColumn: 1, endColumn: 3 });
+		assert.deepEqual(model.getWordAtPosition(new Position(1, 2)), { word: 'ab', startColumn: 1, endColumn: 3 });
+		assert.deepEqual(model.getWordAtPosition(new Position(1, 3)), { word: 'ab', startColumn: 1, endColumn: 3 });
+		assert.deepEqual(model.getWordAtPosition(new Position(1, 4)), { word: 'xx', startColumn: 4, endColumn: 6 });
+		assert.deepEqual(model.getWordAtPosition(new Position(1, 5)), { word: 'xx', startColumn: 4, endColumn: 6 });
+		assert.deepEqual(model.getWordAtPosition(new Position(1, 6)), { word: 'xx', startColumn: 4, endColumn: 6 });
+		assert.deepEqual(model.getWordAtPosition(new Position(1, 7)), { word: 'ab', startColumn: 7, endColumn: 9 });
+	});
 });