Fixes #63822: Keep tokens for empty lines if the language id is not the buffer language id

src/vs/editor/common/model/textModelTokens.ts

@@ -11,7 +11,7 @@ import { Range } from 'vs/editor/common/core/range';
 import { TokenizationResult2 } from 'vs/editor/common/core/token';
 import { ITextBuffer } from 'vs/editor/common/model';
 import { IModelTokensChangedEvent } from 'vs/editor/common/model/textModelEvents';
-import { ColorId, FontStyle, IState, ITokenizationSupport, LanguageId, LanguageIdentifier, MetadataConsts, StandardTokenType } from 'vs/editor/common/modes';
+import { ColorId, FontStyle, IState, ITokenizationSupport, LanguageId, LanguageIdentifier, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes';
 import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 
 function getDefaultMetadata(topLevelLanguageId: LanguageId): number {
@@ -262,8 +262,15 @@ export class ModelLinesTokens {
 		}
 
 		if (lineTextLength === 0) {
-			target._lineTokens = EMPTY_LINE_TOKENS;
-			return;
+			let hasDifferentLanguageId = false;
+			if (tokens && tokens.length > 1) {
+				hasDifferentLanguageId = (TokenMetadata.getLanguageId(tokens[1]) !== topLevelLanguageId);
+			}
+
+			if (!hasDifferentLanguageId) {
+				target._lineTokens = EMPTY_LINE_TOKENS;
+				return;
+			}
 		}
 
 		if (!tokens || tokens.length === 0) {

src/vs/editor/contrib/comment/test/lineCommentCommand.test.ts

@@ -1056,21 +1056,4 @@ suite('Editor Contrib - Line Comment in mixed modes', () => {
 			new Selection(2, 8, 2, 8),
 		);
 	});
-
-	test.skip('issue #63822: Commenting code in JSX tag body from line 0', () => {
-		testLineCommentCommand(
-			[
-				'<div>',
-				'',
-				'</div>',
-			],
-			new Selection(2, 1, 2, 1),
-			[
-				'<div>',
-				'{/* */}',
-				'</div>',
-			],
-			new Selection(2, 1, 2, 1),
-		);
-	});
 });

src/vs/editor/test/common/model/textModelWithTokens.test.ts

@@ -390,6 +390,34 @@ suite('TextModelWithTokens regression tests', () => {
 		model.dispose();
 		registration.dispose();
 	});
+
+	test('issue #63822: Wrong embedded language detected for empty lines', () => {
+		const outerMode = new LanguageIdentifier('outerMode', 3);
+		const innerMode = new LanguageIdentifier('innerMode', 4);
+
+		const tokenizationSupport: ITokenizationSupport = {
+			getInitialState: () => NULL_STATE,
+			tokenize: undefined,
+			tokenize2: (line, state) => {
+				let tokens = new Uint32Array(2);
+				tokens[0] = 0;
+				tokens[1] = (
+					innerMode.id << MetadataConsts.LANGUAGEID_OFFSET
+				) >>> 0;
+				return new TokenizationResult2(tokens, state);
+			}
+		};
+
+		let registration = TokenizationRegistry.register(outerMode.language, tokenizationSupport);
+
+		let model = TextModel.createFromString('A model with one line', undefined, outerMode);
+
+		model.forceTokenization(1);
+		assert.equal(model.getLanguageIdAtPosition(1, 1), innerMode.id);
+
+		model.dispose();
+		registration.dispose();
+	});
 });
 
 suite('TextModel.getLineIndentGuide', () => {