first cut of token_at_position api, https://github.com/microsoft/vscode/issues/91555

src/vs/vscode.proposed.d.ts

@@ -2203,4 +2203,24 @@ declare module 'vscode' {
 
 	//#endregion
 
+
+	//#region https://github.com/microsoft/vscode/issues/91555
+
+	export enum StandardTokenType {
+		Other = 0,
+		Comment = 1,
+		String = 2,
+		RegEx = 4
+	}
+
+	export interface TokenInformation {
+		type: StandardTokenType;
+		range: Range;
+	}
+
+	export namespace languages {
+		export function getTokenInformationAtPosition(document: TextDocument, position: Position): Promise<TokenInformation>;
+	}
+
+	//#endregion
 }

src/vs/workbench/api/browser/mainThreadLanguages.ts

@@ -8,6 +8,9 @@ import { IModeService } from 'vs/editor/common/services/modeService';
 import { IModelService } from 'vs/editor/common/services/modelService';
 import { MainThreadLanguagesShape, MainContext, IExtHostContext } from '../common/extHost.protocol';
 import { extHostNamedCustomer } from 'vs/workbench/api/common/extHostCustomers';
+import { IPosition } from 'vs/editor/common/core/position';
+import { IRange, Range } from 'vs/editor/common/core/range';
+import { StandardTokenType } from 'vs/editor/common/modes';
 
 @extHostNamedCustomer(MainContext.MainThreadLanguages)
 export class MainThreadLanguages implements MainThreadLanguagesShape {
@@ -40,4 +43,19 @@ export class MainThreadLanguages implements MainThreadLanguagesShape {
 		this._modelService.setMode(model, this._modeService.create(languageId));
 		return Promise.resolve(undefined);
 	}
+
+	async $tokensAtPosition(resource: UriComponents, position: IPosition): Promise<undefined | { type: StandardTokenType, range: IRange }> {
+		const uri = URI.revive(resource);
+		const model = this._modelService.getModel(uri);
+		if (!model) {
+			return undefined;
+		}
+		model.tokenizeIfCheap(position.lineNumber);
+		const tokens = model.getLineTokens(position.lineNumber);
+		const idx = tokens.findTokenIndexAtOffset(position.column - 1);
+		return {
+			type: tokens.getStandardTokenType(idx),
+			range: new Range(position.lineNumber, 1 + tokens.getStartOffset(idx), position.lineNumber, 1 + tokens.getEndOffset(idx))
+		};
+	}
 }

src/vs/workbench/api/common/extHost.api.impl.ts

@@ -431,6 +431,10 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 			},
 			setLanguageConfiguration: (language: string, configuration: vscode.LanguageConfiguration): vscode.Disposable => {
 				return extHostLanguageFeatures.setLanguageConfiguration(extension, language, configuration);
+			},
+			getTokenInformationAtPosition(doc: vscode.TextDocument, pos: vscode.Position) {
+				checkProposedApiEnabled(extension);
+				return extHostLanguages.tokenAtPosition(doc, pos);
 			}
 		};
 
@@ -1040,6 +1044,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 			SnippetString: extHostTypes.SnippetString,
 			SourceBreakpoint: extHostTypes.SourceBreakpoint,
 			SourceControlInputBoxValidationType: extHostTypes.SourceControlInputBoxValidationType,
+			StandardTokenType: extHostTypes.StandardTokenType,
 			StatusBarAlignment: extHostTypes.StatusBarAlignment,
 			SymbolInformation: extHostTypes.SymbolInformation,
 			SymbolKind: extHostTypes.SymbolKind,

src/vs/workbench/api/common/extHost.protocol.ts

@@ -393,6 +393,7 @@ export interface MainThreadLanguageFeaturesShape extends IDisposable {
 export interface MainThreadLanguagesShape extends IDisposable {
 	$getLanguages(): Promise<string[]>;
 	$changeLanguage(resource: UriComponents, languageId: string): Promise<void>;
+	$tokensAtPosition(resource: UriComponents, position: IPosition): Promise<undefined | { type: modes.StandardTokenType, range: IRange }>;
 }
 
 export interface MainThreadMessageOptions {

src/vs/workbench/api/common/extHostLanguages.ts

@@ -6,6 +6,8 @@
 import { MainContext, MainThreadLanguagesShape, IMainContext } from './extHost.protocol';
 import type * as vscode from 'vscode';
 import { ExtHostDocuments } from 'vs/workbench/api/common/extHostDocuments';
+import * as typeConvert from 'vs/workbench/api/common/extHostTypeConverters';
+import { StandardTokenType, Range, Position } from 'vs/workbench/api/common/extHostTypes';
 
 export class ExtHostLanguages {
 
@@ -32,4 +34,31 @@ export class ExtHostLanguages {
 		}
 		return data.document;
 	}
+
+	async tokenAtPosition(document: vscode.TextDocument, position: vscode.Position): Promise<vscode.TokenInformation> {
+		const versionNow = document.version;
+		const pos = typeConvert.Position.from(position);
+		const info = await this._proxy.$tokensAtPosition(document.uri, pos);
+		const defaultRange = {
+			type: StandardTokenType.Other,
+			range: document.getWordRangeAtPosition(position) ?? new Range(position.line, position.character, position.line, position.character)
+		};
+		if (!info) {
+			// no result
+			return defaultRange;
+		}
+		const result = {
+			range: typeConvert.Range.to(info.range),
+			type: typeConvert.TokenType.to(info.type)
+		};
+		if (!result.range.contains(<Position>position)) {
+			// bogous result
+			return defaultRange;
+		}
+		if (versionNow !== document.version) {
+			// concurrent change
+			return defaultRange;
+		}
+		return result;
+	}
 }

src/vs/workbench/api/common/extHostTypeConverters.ts

@@ -95,11 +95,22 @@ export namespace Range {
 	}
 }
 
+export namespace TokenType {
+	export function to(type: modes.StandardTokenType): types.StandardTokenType {
+		switch (type) {
+			case modes.StandardTokenType.Comment: return types.StandardTokenType.Comment;
+			case modes.StandardTokenType.Other: return types.StandardTokenType.Other;
+			case modes.StandardTokenType.RegEx: return types.StandardTokenType.RegEx;
+			case modes.StandardTokenType.String: return types.StandardTokenType.String;
+		}
+	}
+}
+
 export namespace Position {
 	export function to(position: IPosition): types.Position {
 		return new types.Position(position.lineNumber - 1, position.column - 1);
 	}
-	export function from(position: types.Position): IPosition {
+	export function from(position: types.Position | vscode.Position): IPosition {
 		return { lineNumber: position.line + 1, column: position.character + 1 };
 	}
 }

src/vs/workbench/api/common/extHostTypes.ts

@@ -2774,3 +2774,9 @@ export class AuthenticationSession implements vscode.AuthenticationSession2 {
 }
 
 //#endregion Authentication
+export enum StandardTokenType {
+	Other = 0,
+	Comment = 1,
+	String = 2,
+	RegEx = 4
+}