Support quickSuggestions in JSDoc Types (#46001)

* Support quickSuggestions in JSDoc Types

Fixes #43715

Requires vscode-textmate 3.3.1 before merging

* Pick up new vscode textmate version

extensions/javascript/package.json

@@ -58,6 +58,9 @@
           "meta.tag.without-attributes.js": "jsx-tags",
           "meta.tag.attributes.js.jsx": "javascriptreact",
           "meta.embedded.expression.js": "javascriptreact"
+        },
+        "tokenTypes": {
+          "entity.name.type.instance.jsdoc": "other"
         }
       },
       {
@@ -69,6 +72,9 @@
           "meta.tag.without-attributes.js": "jsx-tags",
           "meta.tag.attributes.js": "javascript",
           "meta.embedded.expression.js": "javascript"
+        },
+        "tokenTypes": {
+          "entity.name.type.instance.jsdoc": "other"
         }
       },
       {

extensions/typescript-basics/package.json

@@ -42,7 +42,10 @@
       {
         "language": "typescript",
         "scopeName": "source.ts",
-        "path": "./syntaxes/TypeScript.tmLanguage.json"
+        "path": "./syntaxes/TypeScript.tmLanguage.json",
+        "tokenTypes": {
+          "entity.name.type.instance.jsdoc": "other"
+        }
       },
       {
         "language": "typescriptreact",
@@ -53,6 +56,9 @@
           "meta.tag.without-attributes.tsx": "jsx-tags",
           "meta.tag.attributes.tsx": "typescriptreact",
           "meta.embedded.expression.tsx": "typescriptreact"
+        },
+        "tokenTypes": {
+          "entity.name.type.instance.jsdoc": "other"
         }
       }
     ],

package.json

@@ -46,7 +46,7 @@
     "vscode-debugprotocol": "1.27.0",
     "vscode-nsfw": "1.0.17",
     "vscode-ripgrep": "^0.8.1",
-    "vscode-textmate": "^3.2.0",
+    "vscode-textmate": "^3.3.1",
     "vscode-xterm": "3.3.0-beta7",
     "yauzl": "2.8.0"
   },

src/typings/vscode-textmate.d.ts

@@ -37,12 +37,22 @@ declare module "vscode-textmate" {
 	export interface IEmbeddedLanguagesMap {
 		[scopeName: string]: number;
 	}
+	/**
+	 * A map from scope name to a token type.
+	 */
+	export interface ITokenTypeMap {
+		[scopeName: string]: StandardTokenType;
+	}
 	export const enum StandardTokenType {
 		Other = 0,
 		Comment = 1,
 		String = 2,
 		RegEx = 4,
 	}
+	export interface IGrammarConfiguration {
+		embeddedLanguages?: IEmbeddedLanguagesMap;
+		tokenTypes?: ITokenTypeMap;
+	}
 	/**
 	 * The registry that will hold all grammars.
 	 */
@@ -63,6 +73,11 @@ declare module "vscode-textmate" {
 		 * Please do not use language id 0.
 		 */
 		loadGrammarWithEmbeddedLanguages(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap, callback: (err: any, grammar: IGrammar) => void): void;
+		/**
+		 * Load the grammar for `scopeName` and all referenced included grammars asynchronously.
+		 * Please do not use language id 0.
+		 */
+		loadGrammarWithConfiguration(initialScopeName: string, initialLanguage: number, configuration: IGrammarConfiguration, callback: (err: any, grammar: IGrammar) => void): void;
 		/**
 		 * Load the grammar for `scopeName` and all referenced included grammars asynchronously.
 		 */
@@ -75,7 +90,7 @@ declare module "vscode-textmate" {
 		/**
 		 * Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `loadGrammarFromPathSync`.
 		 */
-		grammarForScopeName(scopeName: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap): IGrammar;
+		grammarForScopeName(scopeName: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: ITokenTypeMap): IGrammar;
 	}
 	/**
 	 * A grammar

src/vs/workbench/services/textMate/electron-browser/TMGrammars.ts

@@ -12,11 +12,16 @@ export interface IEmbeddedLanguagesMap {
 	[scopeName: string]: string;
 }
 
+export interface TokenTypesContribution {
+	[scopeName: string]: string;
+}
+
 export interface ITMSyntaxExtensionPoint {
 	language: string;
 	scopeName: string;
 	path: string;
 	embeddedLanguages: IEmbeddedLanguagesMap;
+	tokenTypes: TokenTypesContribution;
 	injectTo: string[];
 }
 
@@ -44,6 +49,13 @@ export const grammarsExtPoint: IExtensionPoint<ITMSyntaxExtensionPoint[]> = Exte
 				description: nls.localize('vscode.extension.contributes.grammars.embeddedLanguages', 'A map of scope name to language id if this grammar contains embedded languages.'),
 				type: 'object'
 			},
+			tokenTypes: {
+				description: nls.localize('vscode.extension.contributes.grammars.tokenTypes', 'A map of scope name to token types.'),
+				type: 'object',
+				additionalProperties: {
+					enum: ['string', 'comment', 'other']
+				}
+			},
 			injectTo: {
 				description: nls.localize('vscode.extension.contributes.grammars.injectTo', 'List of language scope names to which this grammar is injected to.'),
 				type: 'array',

src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts

@@ -14,10 +14,10 @@ import { onUnexpectedError } from 'vs/base/common/errors';
 import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry';
 import { ITokenizationSupport, TokenizationRegistry, IState, LanguageId, TokenMetadata } from 'vs/editor/common/modes';
 import { IModeService } from 'vs/editor/common/services/modeService';
-import { StackElement, IGrammar, Registry, IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2 } from 'vscode-textmate';
+import { StackElement, IGrammar, Registry, IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2, ITokenTypeMap, StandardTokenType } from 'vscode-textmate';
 import { IWorkbenchThemeService, ITokenColorizationRule } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { ITextMateService } from 'vs/workbench/services/textMate/electron-browser/textMateService';
-import { grammarsExtPoint, IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint } from 'vs/workbench/services/textMate/electron-browser/TMGrammars';
+import { grammarsExtPoint, IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint, TokenTypesContribution } from 'vs/workbench/services/textMate/electron-browser/TMGrammars';
 import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
 import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
@@ -36,7 +36,7 @@ export class TMScopeRegistry {
 		this._encounteredLanguages = [];
 	}
 
-	public register(scopeName: string, filePath: string, embeddedLanguages?: IEmbeddedLanguagesMap): void {
+	public register(scopeName: string, filePath: string, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: TokenTypesContribution): void {
 		if (this._scopeNameToLanguageRegistration[scopeName]) {
 			const existingRegistration = this._scopeNameToLanguageRegistration[scopeName];
 			if (existingRegistration.grammarFilePath !== filePath) {
@@ -47,7 +47,7 @@ export class TMScopeRegistry {
 				);
 			}
 		}
-		this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(scopeName, filePath, embeddedLanguages);
+		this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(scopeName, filePath, embeddedLanguages, tokenTypes);
 	}
 
 	public getLanguageRegistration(scopeName: string): TMLanguageRegistration {
@@ -76,8 +76,9 @@ export class TMLanguageRegistration {
 	readonly scopeName: string;
 	readonly grammarFilePath: string;
 	readonly embeddedLanguages: IEmbeddedLanguagesMap;
+	readonly tokenTypes: ITokenTypeMap;
 
-	constructor(scopeName: string, grammarFilePath: string, embeddedLanguages: IEmbeddedLanguagesMap) {
+	constructor(scopeName: string, grammarFilePath: string, embeddedLanguages: IEmbeddedLanguagesMap, tokenTypes: TokenTypesContribution | undefined) {
 		this.scopeName = scopeName;
 		this.grammarFilePath = grammarFilePath;
 
@@ -97,6 +98,27 @@ export class TMLanguageRegistration {
 				this.embeddedLanguages[scope] = language;
 			}
 		}
+
+		this.tokenTypes = Object.create(null);
+		if (tokenTypes) {
+			// If tokenTypes is configured, fill in `this._tokenTypes`
+			const scopes = Object.keys(tokenTypes);
+			for (let i = 0, len = scopes.length; i < len; i++) {
+				const scope = scopes[i];
+				const tokenType = tokenTypes[scope];
+				switch (tokenType) {
+					case 'string':
+						this.tokenTypes[scope] = StandardTokenType.String;
+						break;
+					case 'other':
+						this.tokenTypes[scope] = StandardTokenType.Other;
+						break;
+					case 'comment':
+						this.tokenTypes[scope] = StandardTokenType.Comment;
+						break;
+				}
+			}
+		}
 	}
 }
 
@@ -262,13 +284,18 @@ export class TextMateService implements ITextMateService {
 			return;
 		}
 
+		if (syntax.tokenTypes && !types.isObject(syntax.tokenTypes)) {
+			collector.error(nls.localize('invalid.tokenTypes', "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.tokenTypes)));
+			return;
+		}
+
 		let normalizedAbsolutePath = normalize(join(extensionFolderPath, syntax.path));
 
 		if (normalizedAbsolutePath.indexOf(extensionFolderPath) !== 0) {
 			collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, normalizedAbsolutePath, extensionFolderPath));
 		}
 
-		this._scopeRegistry.register(syntax.scopeName, normalizedAbsolutePath, syntax.embeddedLanguages);
+		this._scopeRegistry.register(syntax.scopeName, normalizedAbsolutePath, syntax.embeddedLanguages, syntax.tokenTypes);
 
 		if (syntax.injectTo) {
 			for (let injectScope of syntax.injectTo) {
@@ -336,7 +363,7 @@ export class TextMateService implements ITextMateService {
 		return this._getOrCreateGrammarRegistry().then((_res) => {
 			const [grammarRegistry, initialState] = _res;
 			return new TPromise<ICreateGrammarResult>((c, e, p) => {
-				grammarRegistry.loadGrammarWithEmbeddedLanguages(scopeName, languageId, embeddedLanguages, (err, grammar) => {
+				grammarRegistry.loadGrammarWithConfiguration(scopeName, languageId, { embeddedLanguages, tokenTypes: languageRegistration.tokenTypes }, (err, grammar) => {
 					if (err) {
 						return e(err);
 					}

yarn.lock

@@ -5920,9 +5920,9 @@ vscode-ripgrep@^0.8.1:
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-0.8.1.tgz#861d2ac97a3764e9f40f305620423efc50632ad1"
 
-vscode-textmate@^3.2.0:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-3.2.0.tgz#87e5ab1ed30463291a73fe28b37a58590a7777dc"
+vscode-textmate@^3.3.1:
+  version "3.3.1"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-3.3.1.tgz#f5b80bca795795e333660c648edba3a715b1e405"
   dependencies:
     fast-plist "^0.1.2"
     oniguruma "^6.0.1"