From 8a04f8fed7edda3d6c54bb3a6364c51a8942834f Mon Sep 17 00:00:00 2001 From: Max Sysoev Date: Fri, 15 Feb 2019 16:54:31 +0100 Subject: [PATCH] Add maxTokenizationLineLength configuration option --- .vscode/settings.json | 2 +- .../common/config/commonEditorConfig.ts | 5 ++++ .../textMate/electron-browser/TMSyntax.ts | 25 +++++++++++-------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index af8b3803709..614a97ab43f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -57,4 +57,4 @@ } ], "git.ignoreLimitWarning": true -} \ No newline at end of file +} diff --git a/src/vs/editor/common/config/commonEditorConfig.ts b/src/vs/editor/common/config/commonEditorConfig.ts index b88b781dbb7..37c71b02eca 100644 --- a/src/vs/editor/common/config/commonEditorConfig.ts +++ b/src/vs/editor/common/config/commonEditorConfig.ts @@ -832,6 +832,11 @@ const editorConfiguration: IConfigurationNode = { 'default': EDITOR_DEFAULTS.contribInfo.lightbulbEnabled, 'description': nls.localize('codeActions', "Enables the code action lightbulb in the editor.") }, + 'editor.maxTokenizationLineLength': { + 'type': 'integer', + 'default': 20_000, + 'description': nls.localize('maxTokenizationLineLength', "Configures length of lines, for which tokenization process will be skipped for performance reasons") + }, 'editor.codeActionsOnSave': { 'type': 'object', 'properties': { diff --git a/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts b/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts index 7248d076a02..8a593401bdb 100644 --- a/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts +++ b/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts @@ -3,7 +3,6 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as nls from 'vs/nls'; import * as dom from 'vs/base/browser/dom'; import { Color } from 'vs/base/common/color'; import { onUnexpectedError } from 'vs/base/common/errors'; @@ -12,18 +11,21 @@ import * as resources from 'vs/base/common/resources'; import * as types from 'vs/base/common/types'; import { URI } from 'vs/base/common/uri'; import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token'; -import { IState, ITokenizationSupport, LanguageId, TokenMetadata, TokenizationRegistry } from 'vs/editor/common/modes'; +import { IState, ITokenizationSupport, LanguageId, TokenizationRegistry, TokenMetadata } from 'vs/editor/common/modes'; import { nullTokenize2 } from 'vs/editor/common/modes/nullMode'; import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization'; import { IModeService } from 'vs/editor/common/services/modeService'; +import * as nls from 'vs/nls'; import { IFileService } from 'vs/platform/files/common/files'; import { ILogService } from 'vs/platform/log/common/log'; import { INotificationService } from 'vs/platform/notification/common/notification'; import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry'; -import { IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint, TokenTypesContribution, grammarsExtPoint } from 'vs/workbench/services/textMate/electron-browser/TMGrammars'; import { ITextMateService } from 'vs/workbench/services/textMate/electron-browser/textMateService'; +import { grammarsExtPoint, IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint, TokenTypesContribution } from 'vs/workbench/services/textMate/electron-browser/TMGrammars'; import { ITokenColorizationRule, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService'; import { IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2, IGrammar, ITokenTypeMap, Registry, StackElement, StandardTokenType } from 'vscode-textmate'; +import { ConfigurationService } from 'vs/platform/configuration/node/configurationService'; +import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { Disposable, IDisposable, dispose } from 'vs/base/common/lifecycle'; export class TMScopeRegistry { @@ -158,7 +160,8 @@ export class TextMateService extends Disposable implements ITextMateService { @IWorkbenchThemeService private readonly _themeService: IWorkbenchThemeService, @IFileService private readonly _fileService: IFileService, @INotificationService private readonly _notificationService: INotificationService, - @ILogService private readonly _logService: ILogService + @ILogService private readonly _logService: ILogService, + @IConfigurationService private readonly _configurationService: ConfigurationService ) { super(); this._styleElement = dom.createStyleSheet(); @@ -226,7 +229,7 @@ export class TextMateService extends Disposable implements ITextMateService { private _registerDefinitionIfAvailable(modeId: string): void { if (this._languageToScope.has(modeId)) { const promise = this._createGrammar(modeId).then((r) => { - return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService); + return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService, this._configurationService); }, e => { onUnexpectedError(e); return null; @@ -442,15 +445,17 @@ class TMTokenization implements ITokenizationSupport { private readonly _containsEmbeddedLanguages: boolean; private readonly _seenLanguages: boolean[]; private readonly _initialState: StackElement; + private _maxTokenizationLineLength: number; private _tokenizationWarningAlreadyShown: boolean; - constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean, @INotificationService private readonly notificationService: INotificationService) { + constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean, @INotificationService private readonly notificationService: INotificationService, @IConfigurationService readonly configurationService: IConfigurationService) { this._scopeRegistry = scopeRegistry; this._languageId = languageId; this._grammar = grammar; this._initialState = initialState; this._containsEmbeddedLanguages = containsEmbeddedLanguages; this._seenLanguages = []; + this._maxTokenizationLineLength = configurationService.getValue('editor.maxTokenizationLineLength'); } public getInitialState(): IState { @@ -466,13 +471,13 @@ class TMTokenization implements ITokenizationSupport { throw new Error('Unexpected: offsetDelta should be 0.'); } - // Do not attempt to tokenize if a line has over 20k - if (line.length >= 20000) { + // Do not attempt to tokenize if a line is too long + if (line.length >= this._maxTokenizationLineLength) { if (!this._tokenizationWarningAlreadyShown) { this._tokenizationWarningAlreadyShown = true; - this.notificationService.warn(nls.localize('too many characters', "Tokenization is skipped for lines longer than 20k characters for performance reasons.")); + this.notificationService.warn(nls.localize('too many characters', 'Tokenization is skipped for long lines for performance reasons. The length of a long line can be set by configuring editor.maxTokenizationLineLength in settings')); } - console.log(`Line (${line.substr(0, 15)}...): longer than 20k characters, tokenization skipped.`); + console.log(`Line (${line.substr(0, 15)}...): longer than ${this._maxTokenizationLineLength} characters, tokenization skipped.`); return nullTokenize2(this._languageId, line, state, offsetDelta); } -- GitLab