Adopt latest vscode-textmate (fixes #84401)

package.json

@@ -52,7 +52,7 @@
     "vscode-proxy-agent": "^0.5.2",
     "vscode-ripgrep": "^1.5.7",
     "vscode-sqlite3": "4.0.9",
-    "vscode-textmate": "^4.3.0",
+    "vscode-textmate": "^4.4.0",
     "xterm": "4.3.0-beta17",
     "xterm-addon-search": "0.4.0-beta4",
     "xterm-addon-web-links": "0.2.1",

remote/package.json

@@ -19,7 +19,7 @@
     "vscode-nsfw": "1.2.8",
     "vscode-proxy-agent": "^0.5.2",
     "vscode-ripgrep": "^1.5.7",
-    "vscode-textmate": "^4.3.0",
+    "vscode-textmate": "^4.4.0",
     "xterm": "4.3.0-beta17",
     "xterm-addon-search": "0.4.0-beta4",
     "xterm-addon-web-links": "0.2.1",

remote/web/package.json

@@ -4,7 +4,7 @@
 	"dependencies": {
 		"onigasm-umd": "^2.2.4",
 		"semver-umd": "^5.5.3",
-		"vscode-textmate": "^4.3.0",
+		"vscode-textmate": "^4.4.0",
 		"xterm": "4.3.0-beta17",
 		"xterm-addon-search": "0.4.0-beta4",
 		"xterm-addon-web-links": "0.2.1",

remote/web/yarn.lock

@@ -24,10 +24,10 @@ semver-umd@^5.5.3:
   resolved "https://registry.yarnpkg.com/semver-umd/-/semver-umd-5.5.3.tgz#b64d7a2d4f5a717b369d56e31940a38e47e34d1e"
   integrity sha512-HOnQrn2iKnVe/xlqCTzMXQdvSz3rPbD0DmQXYuQ+oK1dpptGFfPghonQrx5JHl2O7EJwDqtQnjhE7ME23q6ngw==
 
-vscode-textmate@^4.3.0:
-  version "4.3.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61"
-  integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg==
+vscode-textmate@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305"
+  integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw==
   dependencies:
     oniguruma "^7.2.0"
 

remote/yarn.lock

@@ -399,10 +399,10 @@ vscode-ripgrep@^1.5.7:
   resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.5.7.tgz#acb6b548af488a4bca5d0f1bb5faf761343289ce"
   integrity sha512-/Vsz/+k8kTvui0q3O74pif9FK0nKopgFTiGNVvxicZANxtSA8J8gUE9GQ/4dpi7D/2yI/YVORszwVskFbz46hQ==
 
-vscode-textmate@^4.3.0:
-  version "4.3.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61"
-  integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg==
+vscode-textmate@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305"
+  integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw==
   dependencies:
     oniguruma "^7.2.0"
 

src/typings/vscode-textmate.d.ts

-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-declare module "vscode-textmate" {
-	/**
-	 * A single theme setting.
-	 */
-	export interface IRawThemeSetting {
-		readonly name?: string;
-		readonly scope?: string | string[];
-		readonly settings: {
-			readonly fontStyle?: string;
-			readonly foreground?: string;
-			readonly background?: string;
-		};
-	}
-	/**
-	 * A TextMate theme.
-	 */
-	export interface IRawTheme {
-		readonly name?: string;
-		readonly settings: IRawThemeSetting[];
-	}
-	export interface Thenable<T> extends PromiseLike<T> {
-	}
-	/**
-	 * A registry helper that can locate grammar file paths given scope names.
-	 */
-	export interface RegistryOptions {
-		theme?: IRawTheme;
-		loadGrammar(scopeName: string): Thenable<IRawGrammar | undefined | null>;
-		getInjections?(scopeName: string): string[];
-		getOnigLib?(): Thenable<IOnigLib>;
-	}
-	/**
-	 * A map from scope name to a language id. Please do not use language id 0.
-	 */
-	export interface IEmbeddedLanguagesMap {
-		[scopeName: string]: number;
-	}
-	/**
-	 * A map from selectors to token types.
-	 */
-	export interface ITokenTypeMap {
-		[selector: string]: StandardTokenType;
-	}
-	export const enum StandardTokenType {
-		Other = 0,
-		Comment = 1,
-		String = 2,
-		RegEx = 4,
-	}
-	export interface IGrammarConfiguration {
-		embeddedLanguages?: IEmbeddedLanguagesMap;
-		tokenTypes?: ITokenTypeMap;
-	}
-	/**
-	 * The registry that will hold all grammars.
-	 */
-	export class Registry {
-		private readonly _locator;
-		private readonly _syncRegistry;
-		constructor(locator?: RegistryOptions);
-		/**
-		 * Change the theme. Once called, no previous `ruleStack` should be used anymore.
-		 */
-		setTheme(theme: IRawTheme): void;
-		/**
-		 * Returns a lookup array for color ids.
-		 */
-		getColorMap(): string[];
-		/**
-		 * Load the grammar for `scopeName` and all referenced included grammars asynchronously.
-		 * Please do not use language id 0.
-		 */
-		loadGrammarWithEmbeddedLanguages(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap): Thenable<IGrammar>;
-		/**
-		 * Load the grammar for `scopeName` and all referenced included grammars asynchronously.
-		 * Please do not use language id 0.
-		 */
-		loadGrammarWithConfiguration(initialScopeName: string, initialLanguage: number, configuration: IGrammarConfiguration): Thenable<IGrammar>;
-		/**
-		 * Load the grammar for `scopeName` and all referenced included grammars asynchronously.
-		 */
-		loadGrammar(initialScopeName: string): Thenable<IGrammar>;
-		private _loadGrammar;
-		/**
-		 * Adds a rawGrammar.
-		 */
-		addGrammar(rawGrammar: IRawGrammar, injections?: string[], initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap): Thenable<IGrammar>;
-		/**
-		 * Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `addGrammar`.
-		 */
-		grammarForScopeName(scopeName: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: ITokenTypeMap): Thenable<IGrammar>;
-	}
-	/**
-	 * A grammar
-	 */
-	export interface IGrammar {
-		/**
-		 * Tokenize `lineText` using previous line state `prevState`.
-		 */
-		tokenizeLine(lineText: string, prevState: StackElement | null): ITokenizeLineResult;
-		/**
-		 * Tokenize `lineText` using previous line state `prevState`.
-		 * The result contains the tokens in binary format, resolved with the following information:
-		 *  - language
-		 *  - token type (regex, string, comment, other)
-		 *  - font style
-		 *  - foreground color
-		 *  - background color
-		 * e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET`
-		 */
-		tokenizeLine2(lineText: string, prevState: StackElement | null): ITokenizeLineResult2;
-	}
-	export interface ITokenizeLineResult {
-		readonly tokens: IToken[];
-		/**
-		 * The `prevState` to be passed on to the next line tokenization.
-		 */
-		readonly ruleStack: StackElement;
-	}
-	/**
-	 * Helpers to manage the "collapsed" metadata of an entire StackElement stack.
-	 * The following assumptions have been made:
-	 *  - languageId < 256 => needs 8 bits
-	 *  - unique color count < 512 => needs 9 bits
-	 *
-	 * The binary format is:
-	 * - -------------------------------------------
-	 *     3322 2222 2222 1111 1111 1100 0000 0000
-	 *     1098 7654 3210 9876 5432 1098 7654 3210
-	 * - -------------------------------------------
-	 *     xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx
-	 *     bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL
-	 * - -------------------------------------------
-	 *  - L = LanguageId (8 bits)
-	 *  - T = StandardTokenType (3 bits)
-	 *  - F = FontStyle (3 bits)
-	 *  - f = foreground color (9 bits)
-	 *  - b = background color (9 bits)
-	 */
-	export const enum MetadataConsts {
-		LANGUAGEID_MASK = 255,
-		TOKEN_TYPE_MASK = 1792,
-		FONT_STYLE_MASK = 14336,
-		FOREGROUND_MASK = 8372224,
-		BACKGROUND_MASK = 4286578688,
-		LANGUAGEID_OFFSET = 0,
-		TOKEN_TYPE_OFFSET = 8,
-		FONT_STYLE_OFFSET = 11,
-		FOREGROUND_OFFSET = 14,
-		BACKGROUND_OFFSET = 23,
-	}
-	export interface ITokenizeLineResult2 {
-		/**
-		 * The tokens in binary format. Each token occupies two array indices. For token i:
-		 *  - at offset 2*i => startIndex
-		 *  - at offset 2*i + 1 => metadata
-		 *
-		 */
-		readonly tokens: Uint32Array;
-		/**
-		 * The `prevState` to be passed on to the next line tokenization.
-		 */
-		readonly ruleStack: StackElement;
-	}
-	export interface IToken {
-		startIndex: number;
-		readonly endIndex: number;
-		readonly scopes: string[];
-	}
-	/**
-	 * **IMPORTANT** - Immutable!
-	 */
-	export interface StackElement {
-		_stackElementBrand: void;
-		readonly depth: number;
-		clone(): StackElement;
-		equals(other: StackElement): boolean;
-	}
-	export const INITIAL: StackElement;
-	export const parseRawGrammar: (content: string, filePath?: string) => IRawGrammar;
-	export interface ILocation {
-		readonly filename: string;
-		readonly line: number;
-		readonly char: number;
-	}
-	export interface ILocatable {
-		readonly $vscodeTextmateLocation?: ILocation;
-	}
-	export interface IRawGrammar extends ILocatable {
-		repository: IRawRepository;
-		readonly scopeName: string;
-		readonly patterns: IRawRule[];
-		readonly injections?: {
-			[expression: string]: IRawRule;
-		};
-		readonly injectionSelector?: string;
-		readonly fileTypes?: string[];
-		readonly name?: string;
-		readonly firstLineMatch?: string;
-	}
-	export interface IRawRepositoryMap {
-		[name: string]: IRawRule;
-		$self: IRawRule;
-		$base: IRawRule;
-	}
-	export type IRawRepository = IRawRepositoryMap & ILocatable;
-	export interface IRawRule extends ILocatable {
-		id?: number;
-		readonly include?: string;
-		readonly name?: string;
-		readonly contentName?: string;
-		readonly match?: string;
-		readonly captures?: IRawCaptures;
-		readonly begin?: string;
-		readonly beginCaptures?: IRawCaptures;
-		readonly end?: string;
-		readonly endCaptures?: IRawCaptures;
-		readonly while?: string;
-		readonly whileCaptures?: IRawCaptures;
-		readonly patterns?: IRawRule[];
-		readonly repository?: IRawRepository;
-		readonly applyEndPatternLast?: boolean;
-	}
-	export interface IRawCapturesMap {
-		[captureId: string]: IRawRule;
-	}
-	export type IRawCaptures = IRawCapturesMap & ILocatable;
-	export interface IOnigLib {
-		createOnigScanner(sources: string[]): OnigScanner;
-		createOnigString(sources: string): OnigString;
-	}
-	export interface IOnigCaptureIndex {
-		start: number;
-		end: number;
-		length: number;
-	}
-	export interface IOnigMatch {
-		index: number;
-		captureIndices: IOnigCaptureIndex[];
-		scanner: OnigScanner;
-	}
-	export interface OnigScanner {
-		findNextMatchSync(string: string | OnigString, startPosition: number): IOnigMatch;
-	}
-	export interface OnigString {
-		readonly content: string;
-		readonly dispose?: () => void;
-	}
-
-
-}

src/vs/workbench/contrib/codeEditor/browser/inspectTMScopes/inspectTMScopes.ts

@@ -174,7 +174,7 @@ class InspectTMScopesWidget extends Disposable implements IContentWidget {
 	private readonly _notificationService: INotificationService;
 	private readonly _model: ITextModel;
 	private readonly _domNode: HTMLElement;
-	private readonly _grammar: Promise<IGrammar>;
+	private readonly _grammar: Promise<IGrammar | null>;
 
 	constructor(
 		editor: IActiveCodeEditor,
@@ -212,7 +212,12 @@ class InspectTMScopesWidget extends Disposable implements IContentWidget {
 		dom.clearNode(this._domNode);
 		this._domNode.appendChild(document.createTextNode(nls.localize('inspectTMScopesWidget.loading', "Loading...")));
 		this._grammar.then(
-			(grammar) => this._compute(grammar, position),
+			(grammar) => {
+				if (!grammar) {
+					throw new Error(`Could not find grammar for language!`);
+				}
+				this._compute(grammar, position);
+			},
 			(err) => {
 				this._notificationService.warn(err);
 				setTimeout(() => {

src/vs/workbench/contrib/themes/test/electron-browser/themes.test.contribution.ts

@@ -216,6 +216,9 @@ class Snapper {
 	public captureSyntaxTokens(fileName: string, content: string): Promise<IToken[]> {
 		const modeId = this.modeService.getModeIdByFilepathOrFirstLine(URI.file(fileName));
 		return this.textMateService.createGrammar(modeId!).then((grammar) => {
+			if (!grammar) {
+				return [];
+			}
 			let lines = content.split(/\r\n|\r|\n/);
 
 			let result = this._tokenize(grammar, lines);

src/vs/workbench/services/textMate/browser/abstractTextMateService.ts

@@ -218,6 +218,9 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
 					return null;
 				}
 				const r = await grammarFactory.createGrammar(languageId);
+				if (!r.grammar) {
+					return null;
+				}
 				const tokenization = new TMTokenization(r.grammar, r.initialState, r.containsEmbeddedLanguages);
 				tokenization.onDidEncounterLanguage((languageId) => {
 					if (!this._encounteredLanguages[languageId]) {
@@ -314,7 +317,7 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
 		return true;
 	}
 
-	public async createGrammar(modeId: string): Promise<IGrammar> {
+	public async createGrammar(modeId: string): Promise<IGrammar | null> {
 		const grammarFactory = await this._getOrCreateGrammarFactory();
 		const { grammar } = await grammarFactory.createGrammar(this._modeService.getLanguageIdentifier(modeId)!.id);
 		return grammar;

src/vs/workbench/services/textMate/common/TMGrammarFactory.ts

@@ -18,7 +18,7 @@ interface ITMGrammarFactoryHost {
 
 export interface ICreateGrammarResult {
 	languageId: LanguageId;
-	grammar: IGrammar;
+	grammar: IGrammar | null;
 	initialState: StackElement;
 	containsEmbeddedLanguages: boolean;
 }

src/vs/workbench/services/textMate/common/textMateService.ts

@@ -14,7 +14,7 @@ export interface ITextMateService {
 
 	onDidEncounterLanguage: Event<LanguageId>;
 
-	createGrammar(modeId: string): Promise<IGrammar>;
+	createGrammar(modeId: string): Promise<IGrammar | null>;
 }
 
 // -------------- Types "liberated" from vscode-textmate due to usage in /common/

yarn.lock

@@ -9072,10 +9072,10 @@ vscode-sqlite3@4.0.9:
   dependencies:
     nan "^2.14.0"
 
-vscode-textmate@^4.3.0:
-  version "4.3.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61"
-  integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg==
+vscode-textmate@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305"
+  integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw==
   dependencies:
     oniguruma "^7.2.0"