Add Force Tokenization command

src/vs/editor/contrib/tokenization/tokenization.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as nls from 'vs/nls';
+import { ICodeEditor } from 'vs/editor/browser/editorBrowser';
+import { EditorAction, ServicesAccessor, registerEditorAction } from 'vs/editor/browser/editorExtensions';
+import { StopWatch } from 'vs/base/common/stopwatch';
+
+class ForceRetokenizeAction extends EditorAction {
+	constructor() {
+		super({
+			id: 'editor.action.forceRetokenize',
+			label: nls.localize('forceRetokenize', "Developer: Force Retokenize"),
+			alias: 'Developer: Force Retokenize',
+			precondition: null
+		});
+	}
+
+	public run(accessor: ServicesAccessor, editor: ICodeEditor): void {
+		if (!editor.hasModel()) {
+			return;
+		}
+		const model = editor.getModel();
+		model.flushTokens();
+		const sw = new StopWatch(true);
+		model.forceTokenization(model.getLineCount());
+		sw.stop();
+		console.log(`tokenization took ${sw.elapsed()}`);
+	}
+}
+
+registerEditorAction(ForceRetokenizeAction);

src/vs/editor/editor.all.ts

@@ -36,6 +36,7 @@ import 'vs/editor/contrib/rename/rename';
 import 'vs/editor/contrib/smartSelect/smartSelect';
 import 'vs/editor/contrib/snippet/snippetController2';
 import 'vs/editor/contrib/suggest/suggestController';
+import 'vs/editor/contrib/tokenization/tokenization';
 import 'vs/editor/contrib/toggleTabFocusMode/toggleTabFocusMode';
 import 'vs/editor/contrib/wordHighlighter/wordHighlighter';
 import 'vs/editor/contrib/wordOperations/wordOperations';