Make forced tokenization explicit

src/vs/editor/common/editorCommon.ts

@@ -2041,7 +2041,8 @@ export interface ITokenizedModel extends ITextModel {
 	setMode(languageIdentifier: LanguageIdentifier): void;
 
 	/**
-	 * Returns the true (inner-most) language mode at a given position.
+	 * Returns the real (inner-most) language mode at a given position.
+	 * The result might be inaccurate. Use `forceTokenization` to ensure accurate tokens.
 	 * @internal
 	 */
 	getLanguageIdAtPosition(lineNumber: number, column: number): LanguageId;

src/vs/editor/common/model/textModelWithTokens.ts

@@ -228,10 +228,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		}
 		let { lineNumber, column } = this.validatePosition({ lineNumber: _lineNumber, column: _column });
 
-		this._withModelTokensChangedEventBuilder((eventBuilder) => {
-			this._updateTokensUntilLine(eventBuilder, lineNumber);
-		});
-
 		let lineTokens = this._getLineTokens(lineNumber);
 		let token = lineTokens.findTokenAtOffset(column - 1);
 		return token.languageId;

src/vs/editor/contrib/comment/common/blockCommentCommand.ts

@@ -121,6 +121,7 @@ export class BlockCommentCommand implements editorCommon.ICommand {
 		var endLineNumber = this._selection.endLineNumber;
 		var endColumn = this._selection.endColumn;
 
+		model.forceTokenization(startLineNumber);
 		let languageId = model.getLanguageIdAtPosition(startLineNumber, startColumn);
 		let config = LanguageConfigurationRegistry.getComments(languageId);
 		if (!config || !config.blockCommentStartToken || !config.blockCommentEndToken) {

src/vs/editor/contrib/comment/common/lineCommentCommand.ts

@@ -67,6 +67,7 @@ export class LineCommentCommand implements editorCommon.ICommand {
 		let lines: ILinePreflightData[] = [];
 		for (let i = 0, lineCount = endLineNumber - startLineNumber + 1; i < lineCount; i++) {
 			let lineNumber = startLineNumber + i;
+			model.forceTokenization(lineNumber);
 			let languageId = model.getLanguageIdAtPosition(lineNumber, 1);
 
 			// Find the commentStr for this line, if none is found then bail out: we cannot do line comments
@@ -265,6 +266,7 @@ export class LineCommentCommand implements editorCommon.ICommand {
 	 * Given an unsuccessful analysis, delegate to the block comment command
 	 */
 	private _executeBlockComment(model: editorCommon.ITokenizedModel, builder: editorCommon.IEditOperationBuilder, s: Selection): void {
+		model.forceTokenization(s.startLineNumber);
 		let languageId = model.getLanguageIdAtPosition(s.startLineNumber, s.startColumn);
 		let config = LanguageConfigurationRegistry.getComments(languageId);
 		if (!config || !config.blockCommentStartToken || !config.blockCommentEndToken) {

src/vs/workbench/parts/emmet/node/editorAccessor.ts

@@ -149,6 +149,7 @@ export class EditorAccessor implements emmet.Editor {
 
 	public getSyntaxInternal(overrideUsingProfiles: boolean): string {
 		let position = this._editor.getSelection().getStartPosition();
+		this._editor.getModel().forceTokenization(position.lineNumber);
 		let languageId = this._editor.getModel().getLanguageIdAtPosition(position.lineNumber, position.column);
 		let language = this._languageIdentifierResolver.getLanguageIdentifier(languageId).language;
 		let syntax = language.split('.').pop();

src/vs/workbench/parts/snippets/electron-browser/insertSnippet.ts

@@ -91,6 +91,7 @@ class InsertSnippetAction extends EditorAction {
 			if (langId) {
 				languageId = modeService.getLanguageIdentifier(langId).id;
 			} else {
+				editor.getModel().forceTokenization(lineNumber);
 				languageId = editor.getModel().getLanguageIdAtPosition(lineNumber, column);
 
 				// validate the `languageId` to ensure this is a user

src/vs/workbench/parts/snippets/electron-browser/snippetsService.ts

@@ -77,6 +77,7 @@ class SnippetsService implements ISnippetsService {
 		// validate the `languageId` to ensure this is a user
 		// facing language with a name and the chance to have
 		// snippets, else fall back to the outer language
+		model.forceTokenization(position.lineNumber);
 		let languageId = model.getLanguageIdAtPosition(position.lineNumber, position.column);
 		let { language } = this._modeService.getLanguageIdentifier(languageId);
 		if (!this._modeService.getLanguageName(language)) {