modelServiceImpl.ts 35.5 KB
Newer Older
E
Erich Gamma 已提交
1 2 3 4 5
/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

A
Alex Dima 已提交
6
import { Emitter, Event } from 'vs/base/common/event';
7
import { Disposable, IDisposable, DisposableStore, dispose } from 'vs/base/common/lifecycle';
A
Alex Dima 已提交
8
import * as platform from 'vs/base/common/platform';
A
wip  
Alexandru Dima 已提交
9
import * as errors from 'vs/base/common/errors';
10
import { URI } from 'vs/base/common/uri';
11
import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions';
A
Alex Dima 已提交
12
import { EditOperation } from 'vs/editor/common/core/editOperation';
J
Johannes Rieken 已提交
13
import { Range } from 'vs/editor/common/core/range';
14
import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
15
import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
16
import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
17
import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits } from 'vs/editor/common/modes';
A
Alex Dima 已提交
18
import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
A
Alex Dima 已提交
19
import { ILanguageSelection } from 'vs/editor/common/services/modeService';
20
import { IModelService, DocumentTokensProvider } from 'vs/editor/common/services/modelService';
S
rename  
Sandeep Somavarapu 已提交
21
import { ITextResourcePropertiesService } from 'vs/editor/common/services/textResourceConfigurationService';
A
Alex Dima 已提交
22
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
A
wip  
Alexandru Dima 已提交
23 24
import { RunOnceScheduler } from 'vs/base/common/async';
import { CancellationTokenSource } from 'vs/base/common/cancellation';
25
import { IThemeService } from 'vs/platform/theme/common/themeService';
26
import { ILogService } from 'vs/platform/log/common/log';
27
import { IUndoRedoService, ResourceEditStackSnapshot } from 'vs/platform/undoRedo/common/undoRedo';
28
import { StringSHA1 } from 'vs/base/common/hash';
29
import { EditStackElement, isEditStackElement } from 'vs/editor/common/model/editStack';
30
import { Schemas } from 'vs/base/common/network';
31
import { SemanticTokensProviderStyling, toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling';
32

33
export interface IEditorSemanticHighlightingOptions {
34
	enabled: true | false | 'configuredByTheme';
35 36
}

B
Benjamin Pasero 已提交
37
function MODEL_ID(resource: URI): string {
A
Alex Dima 已提交
38 39 40
	return resource.toString();
}

41 42 43 44 45 46 47 48 49 50 51 52
function computeModelSha1(model: ITextModel): string {
	// compute the sha1
	const shaComputer = new StringSHA1();
	const snapshot = model.createSnapshot();
	let text: string | null;
	while ((text = snapshot.read())) {
		shaComputer.update(text);
	}
	return shaComputer.digest();
}


A
Alex Dima 已提交
53
class ModelData implements IDisposable {
54
	public readonly model: TextModel;
A
Alex Dima 已提交
55 56 57

	private _languageSelection: ILanguageSelection | null;
	private _languageSelectionListener: IDisposable | null;
E
Erich Gamma 已提交
58

59
	private readonly _modelEventListeners = new DisposableStore();
E
Erich Gamma 已提交
60

61
	constructor(
62
		model: TextModel,
A
Alex Dima 已提交
63 64
		onWillDispose: (model: ITextModel) => void,
		onDidChangeLanguage: (model: ITextModel, e: IModelLanguageChangedEvent) => void
65
	) {
E
Erich Gamma 已提交
66
		this.model = model;
A
Alex Dima 已提交
67

A
Alex Dima 已提交
68 69 70
		this._languageSelection = null;
		this._languageSelectionListener = null;

71 72
		this._modelEventListeners.add(model.onWillDispose(() => onWillDispose(model)));
		this._modelEventListeners.add(model.onDidChangeLanguage((e) => onDidChangeLanguage(model, e)));
E
Erich Gamma 已提交
73 74
	}

A
Alex Dima 已提交
75 76 77 78 79 80 81 82 83 84 85
	private _disposeLanguageSelection(): void {
		if (this._languageSelectionListener) {
			this._languageSelectionListener.dispose();
			this._languageSelectionListener = null;
		}
		if (this._languageSelection) {
			this._languageSelection.dispose();
			this._languageSelection = null;
		}
	}

E
Erich Gamma 已提交
86
	public dispose(): void {
87
		this._modelEventListeners.dispose();
A
Alex Dima 已提交
88
		this._disposeLanguageSelection();
A
Alex Dima 已提交
89
	}
E
Erich Gamma 已提交
90

A
Alex Dima 已提交
91 92 93 94 95 96
	public setLanguage(languageSelection: ILanguageSelection): void {
		this._disposeLanguageSelection();
		this._languageSelection = languageSelection;
		this._languageSelectionListener = this._languageSelection.onDidChange(() => this.model.setMode(languageSelection.languageIdentifier));
		this.model.setMode(languageSelection.languageIdentifier);
	}
A
Alex Dima 已提交
97
}
E
Erich Gamma 已提交
98

S
Sandeep Somavarapu 已提交
99 100
interface IRawEditorConfig {
	tabSize?: any;
D
David Lechner 已提交
101
	indentSize?: any;
S
Sandeep Somavarapu 已提交
102 103 104 105 106 107 108
	insertSpaces?: any;
	detectIndentation?: any;
	trimAutoWhitespace?: any;
	creationOptions?: any;
	largeFileOptimizations?: any;
}

109
interface IRawConfig {
S
Sandeep Somavarapu 已提交
110 111
	eol?: any;
	editor?: IRawEditorConfig;
112 113
}

114
const DEFAULT_EOL = (platform.isLinux || platform.isMacintosh) ? DefaultEndOfLine.LF : DefaultEndOfLine.CRLF;
115

116
export interface EditStackPastFutureElements {
117 118 119 120 121 122 123
	past: EditStackElement[];
	future: EditStackElement[];
}

class DisposedModelInfo {
	constructor(
		public readonly uri: URI,
124
		public readonly initialUndoRedoSnapshot: ResourceEditStackSnapshot | null,
125
		public readonly time: number,
126
		public readonly sharesUndoRedoStack: boolean,
127
		public readonly heapSize: number,
128
		public readonly sha1: string,
129
		public readonly versionId: number,
130 131 132 133
		public readonly alternativeVersionId: number,
	) { }
}

A
Alex Dima 已提交
134 135 136 137 138 139 140 141 142
function schemaShouldMaintainUndoRedoElements(resource: URI) {
	return (
		resource.scheme === Schemas.file
		|| resource.scheme === Schemas.vscodeRemote
		|| resource.scheme === Schemas.userData
		|| resource.scheme === 'fake-fs' // for tests
	);
}

A
Alex Dima 已提交
143
export class ModelServiceImpl extends Disposable implements IModelService {
E
Erich Gamma 已提交
144

145 146
	public static MAX_MEMORY_FOR_CLOSED_FILES_UNDO_STACK = 20 * 1024 * 1024;

147
	public _serviceBrand: undefined;
E
Erich Gamma 已提交
148

A
Alex Dima 已提交
149 150 151 152 153 154
	private readonly _onModelAdded: Emitter<ITextModel> = this._register(new Emitter<ITextModel>());
	public readonly onModelAdded: Event<ITextModel> = this._onModelAdded.event;

	private readonly _onModelRemoved: Emitter<ITextModel> = this._register(new Emitter<ITextModel>());
	public readonly onModelRemoved: Event<ITextModel> = this._onModelRemoved.event;

155
	private readonly _onModelModeChanged: Emitter<{ model: ITextModel; oldModeId: string; }> = this._register(new Emitter<{ model: ITextModel; oldModeId: string; }>());
A
Alex Dima 已提交
156
	public readonly onModelModeChanged: Event<{ model: ITextModel; oldModeId: string; }> = this._onModelModeChanged.event;
157

158
	private _modelCreationOptionsByLanguageAndResource: { [languageAndResource: string]: ITextModelCreationOptions; };
A
Alex Dima 已提交
159 160 161 162

	/**
	 * All the models known in the system.
	 */
163
	private readonly _models: { [modelId: string]: ModelData; };
164
	private readonly _disposedModels: Map<string, DisposedModelInfo>;
165
	private _disposedModelsHeapSize: number;
166
	private readonly _semanticStyling: SemanticStyling;
E
Erich Gamma 已提交
167

168
	constructor(
169 170 171 172 173
		@IConfigurationService private readonly _configurationService: IConfigurationService,
		@ITextResourcePropertiesService private readonly _resourcePropertiesService: ITextResourcePropertiesService,
		@IThemeService private readonly _themeService: IThemeService,
		@ILogService private readonly _logService: ILogService,
		@IUndoRedoService private readonly _undoRedoService: IUndoRedoService,
174
	) {
A
Alex Dima 已提交
175
		super();
176
		this._modelCreationOptionsByLanguageAndResource = Object.create(null);
177 178
		this._models = {};
		this._disposedModels = new Map<string, DisposedModelInfo>();
179
		this._disposedModelsHeapSize = 0;
180
		this._semanticStyling = this._register(new SemanticStyling(this._themeService, this._logService));
B
Benjamin Pasero 已提交
181

182
		this._register(this._configurationService.onDidChangeConfiguration(() => this._updateModelOptions()));
183
		this._updateModelOptions();
A
wip  
Alexandru Dima 已提交
184

185
		this._register(new SemanticColoringFeature(this, this._themeService, this._configurationService, this._semanticStyling));
186
	}
J
Joao Moreno 已提交
187

188
	private static _readModelOptions(config: IRawConfig, isForSimpleWidget: boolean): ITextModelCreationOptions {
189
		let tabSize = EDITOR_MODEL_DEFAULTS.tabSize;
190
		if (config.editor && typeof config.editor.tabSize !== 'undefined') {
A
Alex Dima 已提交
191
			const parsedTabSize = parseInt(config.editor.tabSize, 10);
192 193
			if (!isNaN(parsedTabSize)) {
				tabSize = parsedTabSize;
194
			}
A
Alex Dima 已提交
195 196 197
			if (tabSize < 1) {
				tabSize = 1;
			}
198
		}
199

D
David Lechner 已提交
200
		let indentSize = tabSize;
A
Alex Dima 已提交
201
		if (config.editor && typeof config.editor.indentSize !== 'undefined' && config.editor.indentSize !== 'tabSize') {
A
Alex Dima 已提交
202
			const parsedIndentSize = parseInt(config.editor.indentSize, 10);
D
David Lechner 已提交
203 204 205 206 207 208 209 210
			if (!isNaN(parsedIndentSize)) {
				indentSize = parsedIndentSize;
			}
			if (indentSize < 1) {
				indentSize = 1;
			}
		}

211
		let insertSpaces = EDITOR_MODEL_DEFAULTS.insertSpaces;
212 213 214
		if (config.editor && typeof config.editor.insertSpaces !== 'undefined') {
			insertSpaces = (config.editor.insertSpaces === 'false' ? false : Boolean(config.editor.insertSpaces));
		}
215

216
		let newDefaultEOL = DEFAULT_EOL;
S
Sandeep Somavarapu 已提交
217
		const eol = config.eol;
218
		if (eol === '\r\n') {
219
			newDefaultEOL = DefaultEndOfLine.CRLF;
220
		} else if (eol === '\n') {
221
			newDefaultEOL = DefaultEndOfLine.LF;
222
		}
223

224
		let trimAutoWhitespace = EDITOR_MODEL_DEFAULTS.trimAutoWhitespace;
225 226 227
		if (config.editor && typeof config.editor.trimAutoWhitespace !== 'undefined') {
			trimAutoWhitespace = (config.editor.trimAutoWhitespace === 'false' ? false : Boolean(config.editor.trimAutoWhitespace));
		}
228

229
		let detectIndentation = EDITOR_MODEL_DEFAULTS.detectIndentation;
230 231 232
		if (config.editor && typeof config.editor.detectIndentation !== 'undefined') {
			detectIndentation = (config.editor.detectIndentation === 'false' ? false : Boolean(config.editor.detectIndentation));
		}
233

234 235 236
		let largeFileOptimizations = EDITOR_MODEL_DEFAULTS.largeFileOptimizations;
		if (config.editor && typeof config.editor.largeFileOptimizations !== 'undefined') {
			largeFileOptimizations = (config.editor.largeFileOptimizations === 'false' ? false : Boolean(config.editor.largeFileOptimizations));
237 238
		}

239
		return {
240
			isForSimpleWidget: isForSimpleWidget,
241
			tabSize: tabSize,
D
David Lechner 已提交
242
			indentSize: indentSize,
243 244 245
			insertSpaces: insertSpaces,
			detectIndentation: detectIndentation,
			defaultEOL: newDefaultEOL,
246
			trimAutoWhitespace: trimAutoWhitespace,
247
			largeFileOptimizations: largeFileOptimizations
248
		};
E
Erich Gamma 已提交
249 250
	}

251 252 253 254 255 256 257 258 259 260 261
	private _getEOL(resource: URI | undefined, language: string): string {
		if (resource) {
			return this._resourcePropertiesService.getEOL(resource, language);
		}
		const eol = this._configurationService.getValue<string>('files.eol', { overrideIdentifier: language });
		if (eol && eol !== 'auto') {
			return eol;
		}
		return platform.OS === platform.OperatingSystem.Linux || platform.OS === platform.OperatingSystem.Macintosh ? '\n' : '\r\n';
	}

262 263 264 265
	private _shouldRestoreUndoStack(): boolean {
		const result = this._configurationService.getValue<boolean>('files.restoreUndoStack');
		if (typeof result === 'boolean') {
			return result;
266
		}
267
		return true;
268 269
	}

A
Alex Dima 已提交
270
	public getCreationOptions(language: string, resource: URI | undefined, isForSimpleWidget: boolean): ITextModelCreationOptions {
271
		let creationOptions = this._modelCreationOptionsByLanguageAndResource[language + resource];
272
		if (!creationOptions) {
S
Sandeep Somavarapu 已提交
273
			const editor = this._configurationService.getValue<IRawEditorConfig>('editor', { overrideIdentifier: language, resource });
274
			const eol = this._getEOL(resource, language);
S
Sandeep Somavarapu 已提交
275
			creationOptions = ModelServiceImpl._readModelOptions({ editor, eol }, isForSimpleWidget);
276
			this._modelCreationOptionsByLanguageAndResource[language + resource] = creationOptions;
277 278
		}
		return creationOptions;
279 280
	}

281
	private _updateModelOptions(): void {
A
Alex Dima 已提交
282
		const oldOptionsByLanguageAndResource = this._modelCreationOptionsByLanguageAndResource;
283
		this._modelCreationOptionsByLanguageAndResource = Object.create(null);
284

285
		// Update options on all models
A
Alex Dima 已提交
286
		const keys = Object.keys(this._models);
A
Alex Dima 已提交
287
		for (let i = 0, len = keys.length; i < len; i++) {
A
Alex Dima 已提交
288 289
			const modelId = keys[i];
			const modelData = this._models[modelId];
290
			const language = modelData.model.getLanguageIdentifier().language;
291 292
			const uri = modelData.model.uri;
			const oldOptions = oldOptionsByLanguageAndResource[language + uri];
293
			const newOptions = this.getCreationOptions(language, uri, modelData.model.isForSimpleWidget);
294 295 296
			ModelServiceImpl._setModelOptionsForModel(modelData.model, newOptions, oldOptions);
		}
	}
297

A
Alex Dima 已提交
298
	private static _setModelOptionsForModel(model: ITextModel, newOptions: ITextModelCreationOptions, currentOptions: ITextModelCreationOptions): void {
299 300 301 302
		if (currentOptions && currentOptions.defaultEOL !== newOptions.defaultEOL && model.getLineCount() === 1) {
			model.setEOL(newOptions.defaultEOL === DefaultEndOfLine.LF ? EndOfLineSequence.LF : EndOfLineSequence.CRLF);
		}

303 304 305 306
		if (currentOptions
			&& (currentOptions.detectIndentation === newOptions.detectIndentation)
			&& (currentOptions.insertSpaces === newOptions.insertSpaces)
			&& (currentOptions.tabSize === newOptions.tabSize)
A
Alex Dima 已提交
307
			&& (currentOptions.indentSize === newOptions.indentSize)
308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
			&& (currentOptions.trimAutoWhitespace === newOptions.trimAutoWhitespace)
		) {
			// Same indent opts, no need to touch the model
			return;
		}

		if (newOptions.detectIndentation) {
			model.detectIndentation(newOptions.insertSpaces, newOptions.tabSize);
			model.updateOptions({
				trimAutoWhitespace: newOptions.trimAutoWhitespace
			});
		} else {
			model.updateOptions({
				insertSpaces: newOptions.insertSpaces,
				tabSize: newOptions.tabSize,
A
Alex Dima 已提交
323
				indentSize: newOptions.indentSize,
324 325
				trimAutoWhitespace: newOptions.trimAutoWhitespace
			});
326
		}
327 328
	}

E
Erich Gamma 已提交
329 330
	// --- begin IModelService

331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
	private _insertDisposedModel(disposedModelData: DisposedModelInfo): void {
		this._disposedModels.set(MODEL_ID(disposedModelData.uri), disposedModelData);
		this._disposedModelsHeapSize += disposedModelData.heapSize;
	}

	private _removeDisposedModel(resource: URI): DisposedModelInfo | undefined {
		const disposedModelData = this._disposedModels.get(MODEL_ID(resource));
		if (disposedModelData) {
			this._disposedModelsHeapSize -= disposedModelData.heapSize;
		}
		this._disposedModels.delete(MODEL_ID(resource));
		return disposedModelData;
	}

	private _ensureDisposedModelsHeapSize(maxModelsHeapSize: number): void {
		if (this._disposedModelsHeapSize > maxModelsHeapSize) {
			// we must remove some old undo stack elements to free up some memory
			const disposedModels: DisposedModelInfo[] = [];
349 350 351 352 353
			this._disposedModels.forEach(entry => {
				if (!entry.sharesUndoRedoStack) {
					disposedModels.push(entry);
				}
			});
354 355 356 357
			disposedModels.sort((a, b) => a.time - b.time);
			while (disposedModels.length > 0 && this._disposedModelsHeapSize > maxModelsHeapSize) {
				const disposedModel = disposedModels.shift()!;
				this._removeDisposedModel(disposedModel.uri);
358 359 360
				if (disposedModel.initialUndoRedoSnapshot !== null) {
					this._undoRedoService.restoreSnapshot(disposedModel.initialUndoRedoSnapshot);
				}
361 362 363 364
			}
		}
	}

A
Alex Dima 已提交
365
	private _createModelData(value: string | ITextBufferFactory, languageIdentifier: LanguageIdentifier, resource: URI | undefined, isForSimpleWidget: boolean): ModelData {
A
Alex Dima 已提交
366
		// create & save the model
367
		const options = this.getCreationOptions(languageIdentifier.language, resource, isForSimpleWidget);
368
		const model: TextModel = new TextModel(value, options, languageIdentifier, resource, this._undoRedoService);
369
		if (resource && this._disposedModels.has(MODEL_ID(resource))) {
370
			const disposedModelData = this._removeDisposedModel(resource)!;
371
			const elements = this._undoRedoService.getElements(resource);
372 373
			const sha1IsEqual = (computeModelSha1(model) === disposedModelData.sha1);
			if (sha1IsEqual || disposedModelData.sharesUndoRedoStack) {
374
				for (const element of elements.past) {
375 376 377
					if (isEditStackElement(element) && element.matchesResource(resource)) {
						element.setModel(model);
					}
378 379
				}
				for (const element of elements.future) {
380 381 382 383 384 385 386 387
					if (isEditStackElement(element) && element.matchesResource(resource)) {
						element.setModel(model);
					}
				}
				this._undoRedoService.setElementsValidFlag(resource, true, (element) => (isEditStackElement(element) && element.matchesResource(resource)));
				if (sha1IsEqual) {
					model._overwriteVersionId(disposedModelData.versionId);
					model._overwriteAlternativeVersionId(disposedModelData.alternativeVersionId);
388
					model._overwriteInitialUndoRedoSnapshot(disposedModelData.initialUndoRedoSnapshot);
389 390
				}
			} else {
391 392 393
				if (disposedModelData.initialUndoRedoSnapshot !== null) {
					this._undoRedoService.restoreSnapshot(disposedModelData.initialUndoRedoSnapshot);
				}
394 395
			}
		}
396
		const modelId = MODEL_ID(model.uri);
E
Erich Gamma 已提交
397 398 399

		if (this._models[modelId]) {
			// There already exists a model with this id => this is a programmer error
400
			throw new Error('ModelService: Cannot add model because it already exists!');
E
Erich Gamma 已提交
401 402
		}

403
		const modelData = new ModelData(
404 405 406 407
			model,
			(model) => this._onWillDispose(model),
			(model, e) => this._onDidChangeLanguage(model, e)
		);
A
Alex Dima 已提交
408
		this._models[modelId] = modelData;
E
Erich Gamma 已提交
409

A
Alex Dima 已提交
410
		return modelData;
E
Erich Gamma 已提交
411 412
	}

413
	public updateModel(model: ITextModel, value: string | ITextBufferFactory): void {
414
		const options = this.getCreationOptions(model.getLanguageIdentifier().language, model.uri, model.isForSimpleWidget);
415
		const textBuffer = createTextBuffer(value, options.defaultEOL);
416 417

		// Return early if the text is already set in that form
418
		if (model.equalsTextBuffer(textBuffer)) {
419 420
			return;
		}
421 422

		// Otherwise find a diff between the values and update model
423
		model.pushStackElement();
A
Alex Dima 已提交
424
		model.pushEOL(textBuffer.getEOL() === '\r\n' ? EndOfLineSequence.CRLF : EndOfLineSequence.LF);
425
		model.pushEditOperations(
426
			[],
427
			ModelServiceImpl._computeEdits(model, textBuffer),
428
			() => []
429
		);
430
		model.pushStackElement();
431 432
	}

433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
	private static _commonPrefix(a: ILineSequence, aLen: number, aDelta: number, b: ILineSequence, bLen: number, bDelta: number): number {
		const maxResult = Math.min(aLen, bLen);

		let result = 0;
		for (let i = 0; i < maxResult && a.getLineContent(aDelta + i) === b.getLineContent(bDelta + i); i++) {
			result++;
		}
		return result;
	}

	private static _commonSuffix(a: ILineSequence, aLen: number, aDelta: number, b: ILineSequence, bLen: number, bDelta: number): number {
		const maxResult = Math.min(aLen, bLen);

		let result = 0;
		for (let i = 0; i < maxResult && a.getLineContent(aDelta + aLen - i) === b.getLineContent(bDelta + bLen - i); i++) {
			result++;
		}
		return result;
	}

453 454 455
	/**
	 * Compute edits to bring `model` to the state of `textSource`.
	 */
456
	public static _computeEdits(model: ITextModel, textBuffer: ITextBuffer): IIdentifiedSingleEditOperation[] {
457
		const modelLineCount = model.getLineCount();
458 459
		const textBufferLineCount = textBuffer.getLineCount();
		const commonPrefix = this._commonPrefix(model, modelLineCount, 1, textBuffer, textBufferLineCount, 1);
460

461 462 463 464
		if (modelLineCount === textBufferLineCount && commonPrefix === modelLineCount) {
			// equality case
			return [];
		}
465

466
		const commonSuffix = this._commonSuffix(model, modelLineCount - commonPrefix, commonPrefix, textBuffer, textBufferLineCount - commonPrefix, commonPrefix);
467

A
Alex Dima 已提交
468 469
		let oldRange: Range;
		let newRange: Range;
470 471 472 473 474 475 476 477 478
		if (commonSuffix > 0) {
			oldRange = new Range(commonPrefix + 1, 1, modelLineCount - commonSuffix + 1, 1);
			newRange = new Range(commonPrefix + 1, 1, textBufferLineCount - commonSuffix + 1, 1);
		} else if (commonPrefix > 0) {
			oldRange = new Range(commonPrefix, model.getLineMaxColumn(commonPrefix), modelLineCount, model.getLineMaxColumn(modelLineCount));
			newRange = new Range(commonPrefix, 1 + textBuffer.getLineLength(commonPrefix), textBufferLineCount, 1 + textBuffer.getLineLength(textBufferLineCount));
		} else {
			oldRange = new Range(1, 1, modelLineCount, model.getLineMaxColumn(modelLineCount));
			newRange = new Range(1, 1, textBufferLineCount, 1 + textBuffer.getLineLength(textBufferLineCount));
479 480
		}

481
		return [EditOperation.replaceMove(oldRange, textBuffer.getValueInRange(newRange, EndOfLinePreference.TextDefined))];
482 483
	}

A
Alex Dima 已提交
484
	public createModel(value: string | ITextBufferFactory, languageSelection: ILanguageSelection | null, resource?: URI, isForSimpleWidget: boolean = false): ITextModel {
485 486
		let modelData: ModelData;

A
Alex Dima 已提交
487 488 489
		if (languageSelection) {
			modelData = this._createModelData(value, languageSelection.languageIdentifier, resource, isForSimpleWidget);
			this.setMode(modelData.model, languageSelection);
490
		} else {
A
Alex Dima 已提交
491
			modelData = this._createModelData(value, PLAINTEXT_LANGUAGE_IDENTIFIER, resource, isForSimpleWidget);
492
		}
E
Erich Gamma 已提交
493

A
Alex Dima 已提交
494
		this._onModelAdded.fire(modelData.model);
E
Erich Gamma 已提交
495

A
Alex Dima 已提交
496
		return modelData.model;
E
Erich Gamma 已提交
497 498
	}

A
Alex Dima 已提交
499 500
	public setMode(model: ITextModel, languageSelection: ILanguageSelection): void {
		if (!languageSelection) {
501 502
			return;
		}
A
Alex Dima 已提交
503
		const modelData = this._models[MODEL_ID(model.uri)];
A
Alex Dima 已提交
504 505
		if (!modelData) {
			return;
506
		}
A
Alex Dima 已提交
507
		modelData.setLanguage(languageSelection);
508 509
	}

J
Johannes Rieken 已提交
510
	public destroyModel(resource: URI): void {
A
Alex Dima 已提交
511
		// We need to support that not all models get disposed through this service (i.e. model.dispose() should work!)
A
Alex Dima 已提交
512
		const modelData = this._models[MODEL_ID(resource)];
A
Alex Dima 已提交
513 514
		if (!modelData) {
			return;
E
Erich Gamma 已提交
515
		}
516
		modelData.model.dispose();
E
Erich Gamma 已提交
517 518
	}

A
Alex Dima 已提交
519
	public getModels(): ITextModel[] {
A
Alex Dima 已提交
520
		const ret: ITextModel[] = [];
A
Alex Dima 已提交
521

A
Alex Dima 已提交
522
		const keys = Object.keys(this._models);
A
Alex Dima 已提交
523
		for (let i = 0, len = keys.length; i < len; i++) {
A
Alex Dima 已提交
524
			const modelId = keys[i];
A
Alex Dima 已提交
525
			ret.push(this._models[modelId].model);
E
Erich Gamma 已提交
526
		}
A
Alex Dima 已提交
527

E
Erich Gamma 已提交
528 529 530
		return ret;
	}

A
Alex Dima 已提交
531
	public getModel(resource: URI): ITextModel | null {
A
Alex Dima 已提交
532 533
		const modelId = MODEL_ID(resource);
		const modelData = this._models[modelId];
A
Alex Dima 已提交
534 535
		if (!modelData) {
			return null;
E
Erich Gamma 已提交
536
		}
A
Alex Dima 已提交
537
		return modelData.model;
E
Erich Gamma 已提交
538 539
	}

540 541 542 543
	public getSemanticTokensProviderStyling(provider: DocumentTokensProvider): SemanticTokensProviderStyling {
		return this._semanticStyling.get(provider);
	}

E
Erich Gamma 已提交
544 545
	// --- end IModelService

A
Alex Dima 已提交
546
	private _onWillDispose(model: ITextModel): void {
A
Alex Dima 已提交
547 548
		const modelId = MODEL_ID(model.uri);
		const modelData = this._models[modelId];
A
Alex Dima 已提交
549

550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
		const sharesUndoRedoStack = (this._undoRedoService.getUriComparisonKey(model.uri) !== model.uri.toString());
		let maintainUndoRedoStack = false;
		let heapSize = 0;
		if (sharesUndoRedoStack || (this._shouldRestoreUndoStack() && schemaShouldMaintainUndoRedoElements(model.uri))) {
			const elements = this._undoRedoService.getElements(model.uri);
			if (elements.past.length > 0 || elements.future.length > 0) {
				for (const element of elements.past) {
					if (isEditStackElement(element) && element.matchesResource(model.uri)) {
						maintainUndoRedoStack = true;
						heapSize += element.heapSize(model.uri);
						element.setModel(model.uri); // remove reference from text buffer instance
					}
				}
				for (const element of elements.future) {
					if (isEditStackElement(element) && element.matchesResource(model.uri)) {
						maintainUndoRedoStack = true;
						heapSize += element.heapSize(model.uri);
						element.setModel(model.uri); // remove reference from text buffer instance
					}
				}
			}
		}

		const maxMemory = ModelServiceImpl.MAX_MEMORY_FOR_CLOSED_FILES_UNDO_STACK;
		if (!maintainUndoRedoStack) {
			if (!sharesUndoRedoStack) {
				const initialUndoRedoSnapshot = modelData.model.getInitialUndoRedoSnapshot();
				if (initialUndoRedoSnapshot !== null) {
					this._undoRedoService.restoreSnapshot(initialUndoRedoSnapshot);
				}
			}
		} else if (!sharesUndoRedoStack && heapSize > maxMemory) {
			// the undo stack for this file would never fit in the configured memory, so don't bother with it.
			const initialUndoRedoSnapshot = modelData.model.getInitialUndoRedoSnapshot();
			if (initialUndoRedoSnapshot !== null) {
				this._undoRedoService.restoreSnapshot(initialUndoRedoSnapshot);
			}
		} else {
			this._ensureDisposedModelsHeapSize(maxMemory - heapSize);
			// We only invalidate the elements, but they remain in the undo-redo service.
			this._undoRedoService.setElementsValidFlag(model.uri, false, (element) => (isEditStackElement(element) && element.matchesResource(model.uri)));
			this._insertDisposedModel(new DisposedModelInfo(model.uri, modelData.model.getInitialUndoRedoSnapshot(), Date.now(), sharesUndoRedoStack, heapSize, computeModelSha1(model), model.getVersionId(), model.getAlternativeVersionId()));
		}

A
Alex Dima 已提交
594 595 596
		delete this._models[modelId];
		modelData.dispose();

597 598 599
		// clean up cache
		delete this._modelCreationOptionsByLanguageAndResource[model.getLanguageIdentifier().language + model.uri];

A
Alex Dima 已提交
600 601 602
		this._onModelRemoved.fire(model);
	}

A
Alex Dima 已提交
603
	private _onDidChangeLanguage(model: ITextModel, e: IModelLanguageChangedEvent): void {
604 605
		const oldModeId = e.oldLanguage;
		const newModeId = model.getLanguageIdentifier().language;
606 607
		const oldOptions = this.getCreationOptions(oldModeId, model.uri, model.isForSimpleWidget);
		const newOptions = this.getCreationOptions(newModeId, model.uri, model.isForSimpleWidget);
608 609
		ModelServiceImpl._setModelOptionsForModel(model, newOptions, oldOptions);
		this._onModelModeChanged.fire({ model, oldModeId });
E
Erich Gamma 已提交
610 611
	}
}
612 613 614 615

export interface ILineSequence {
	getLineContent(lineNumber: number): string;
}
A
wip  
Alexandru Dima 已提交
616

617 618 619
export const SEMANTIC_HIGHLIGHTING_SETTING_ID = 'editor.semanticHighlighting';

export function isSemanticColoringEnabled(model: ITextModel, themeService: IThemeService, configurationService: IConfigurationService): boolean {
M
Martin Aeschlimann 已提交
620
	const setting = configurationService.getValue<IEditorSemanticHighlightingOptions>(SEMANTIC_HIGHLIGHTING_SETTING_ID, { overrideIdentifier: model.getLanguageIdentifier().language, resource: model.uri })?.enabled;
621 622
	if (typeof setting === 'boolean') {
		return setting;
623
	}
624
	return themeService.getColorTheme().semanticHighlighting;
625
}
626

627
class SemanticColoringFeature extends Disposable {
628

629 630
	private readonly _watchers: Record<string, ModelSemanticColoring>;
	private readonly _semanticStyling: SemanticStyling;
A
wip  
Alexandru Dima 已提交
631

632
	constructor(modelService: IModelService, themeService: IThemeService, configurationService: IConfigurationService, semanticStyling: SemanticStyling) {
A
wip  
Alexandru Dima 已提交
633 634
		super();
		this._watchers = Object.create(null);
635
		this._semanticStyling = semanticStyling;
636 637

		const register = (model: ITextModel) => {
638
			this._watchers[model.uri.toString()] = new ModelSemanticColoring(model, themeService, this._semanticStyling);
639 640 641 642 643
		};
		const deregister = (model: ITextModel, modelSemanticColoring: ModelSemanticColoring) => {
			modelSemanticColoring.dispose();
			delete this._watchers[model.uri.toString()];
		};
644 645 646
		const handleSettingOrThemeChange = () => {
			for (let model of modelService.getModels()) {
				const curr = this._watchers[model.uri.toString()];
647
				if (isSemanticColoringEnabled(model, themeService, configurationService)) {
648 649 650 651 652 653 654 655 656 657
					if (!curr) {
						register(model);
					}
				} else {
					if (curr) {
						deregister(model, curr);
					}
				}
			}
		};
658
		this._register(modelService.onModelAdded((model) => {
659
			if (isSemanticColoringEnabled(model, themeService, configurationService)) {
660 661
				register(model);
			}
A
wip  
Alexandru Dima 已提交
662 663
		}));
		this._register(modelService.onModelRemoved((model) => {
664 665 666 667
			const curr = this._watchers[model.uri.toString()];
			if (curr) {
				deregister(model, curr);
			}
A
wip  
Alexandru Dima 已提交
668
		}));
669
		this._register(configurationService.onDidChangeConfiguration(e => {
670
			if (e.affectsConfiguration(SEMANTIC_HIGHLIGHTING_SETTING_ID)) {
671
				handleSettingOrThemeChange();
672
			}
673 674
		}));
		this._register(themeService.onDidColorThemeChange(handleSettingOrThemeChange));
A
wip  
Alexandru Dima 已提交
675 676 677
	}
}

678
class SemanticStyling extends Disposable {
679

680
	private _caches: WeakMap<DocumentTokensProvider, SemanticTokensProviderStyling>;
681 682

	constructor(
683 684
		private readonly _themeService: IThemeService,
		private readonly _logService: ILogService
685
	) {
686
		super();
687
		this._caches = new WeakMap<DocumentTokensProvider, SemanticTokensProviderStyling>();
688
		this._register(this._themeService.onDidColorThemeChange(() => {
689
			this._caches = new WeakMap<DocumentTokensProvider, SemanticTokensProviderStyling>();
690
		}));
691 692
	}

693
	public get(provider: DocumentTokensProvider): SemanticTokensProviderStyling {
694
		if (!this._caches.has(provider)) {
695
			this._caches.set(provider, new SemanticTokensProviderStyling(provider.getLegend(), this._themeService, this._logService));
696 697 698 699 700
		}
		return this._caches.get(provider)!;
	}
}

701 702
class SemanticTokensResponse {
	constructor(
703
		private readonly _provider: DocumentSemanticTokensProvider,
704 705 706 707 708
		public readonly resultId: string | undefined,
		public readonly data: Uint32Array
	) { }

	public dispose(): void {
709
		this._provider.releaseDocumentSemanticTokens(this.resultId);
710 711 712
	}
}

A
wip  
Alexandru Dima 已提交
713 714
class ModelSemanticColoring extends Disposable {

715
	private _isDisposed: boolean;
A
wip  
Alexandru Dima 已提交
716
	private readonly _model: ITextModel;
717
	private readonly _semanticStyling: SemanticStyling;
A
Alex Dima 已提交
718 719 720 721
	private readonly _fetchDocumentSemanticTokens: RunOnceScheduler;
	private _currentDocumentResponse: SemanticTokensResponse | null;
	private _currentDocumentRequestCancellationTokenSource: CancellationTokenSource | null;
	private _documentProvidersChangeListeners: IDisposable[];
A
wip  
Alexandru Dima 已提交
722

723
	constructor(model: ITextModel, themeService: IThemeService, stylingProvider: SemanticStyling) {
A
wip  
Alexandru Dima 已提交
724
		super();
A
wip  
Alexandru Dima 已提交
725

726
		this._isDisposed = false;
A
wip  
Alexandru Dima 已提交
727
		this._model = model;
728
		this._semanticStyling = stylingProvider;
A
Alex Dima 已提交
729 730 731 732
		this._fetchDocumentSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchDocumentSemanticTokensNow(), 300));
		this._currentDocumentResponse = null;
		this._currentDocumentRequestCancellationTokenSource = null;
		this._documentProvidersChangeListeners = [];
A
wip  
Alexandru Dima 已提交
733

734
		this._register(this._model.onDidChangeContent(() => {
A
Alex Dima 已提交
735 736
			if (!this._fetchDocumentSemanticTokens.isScheduled()) {
				this._fetchDocumentSemanticTokens.schedule();
737 738
			}
		}));
A
Alex Dima 已提交
739 740 741
		const bindDocumentChangeListeners = () => {
			dispose(this._documentProvidersChangeListeners);
			this._documentProvidersChangeListeners = [];
742 743
			for (const provider of DocumentSemanticTokensProviderRegistry.all(model)) {
				if (typeof provider.onDidChange === 'function') {
A
Alex Dima 已提交
744
					this._documentProvidersChangeListeners.push(provider.onDidChange(() => this._fetchDocumentSemanticTokens.schedule(0)));
745 746 747
				}
			}
		};
A
Alex Dima 已提交
748
		bindDocumentChangeListeners();
749
		this._register(DocumentSemanticTokensProviderRegistry.onDidChange(() => {
A
Alex Dima 已提交
750 751
			bindDocumentChangeListeners();
			this._fetchDocumentSemanticTokens.schedule();
752 753
		}));

754 755
		this._register(themeService.onDidColorThemeChange(_ => {
			// clear out existing tokens
A
Alex Dima 已提交
756 757
			this._setDocumentSemanticTokens(null, null, null, []);
			this._fetchDocumentSemanticTokens.schedule();
758 759
		}));

A
Alex Dima 已提交
760
		this._fetchDocumentSemanticTokens.schedule(0);
A
wip  
Alexandru Dima 已提交
761 762 763
	}

	public dispose(): void {
A
Alex Dima 已提交
764 765 766
		if (this._currentDocumentResponse) {
			this._currentDocumentResponse.dispose();
			this._currentDocumentResponse = null;
A
wip  
Alexandru Dima 已提交
767
		}
A
Alex Dima 已提交
768 769 770
		if (this._currentDocumentRequestCancellationTokenSource) {
			this._currentDocumentRequestCancellationTokenSource.cancel();
			this._currentDocumentRequestCancellationTokenSource = null;
A
wip  
Alexandru Dima 已提交
771
		}
A
Alex Dima 已提交
772
		this._setDocumentSemanticTokens(null, null, null, []);
773 774
		this._isDisposed = true;

A
wip  
Alexandru Dima 已提交
775
		super.dispose();
A
wip  
Alexandru Dima 已提交
776 777
	}

A
Alex Dima 已提交
778 779
	private _fetchDocumentSemanticTokensNow(): void {
		if (this._currentDocumentRequestCancellationTokenSource) {
A
wip  
Alexandru Dima 已提交
780 781 782 783 784
			// there is already a request running, let it finish...
			return;
		}
		const provider = this._getSemanticColoringProvider();
		if (!provider) {
785 786 787 788
			if (this._currentDocumentResponse) {
				// there are semantic tokens set
				this._model.setSemanticTokens(null, false);
			}
A
wip  
Alexandru Dima 已提交
789 790
			return;
		}
A
Alex Dima 已提交
791
		this._currentDocumentRequestCancellationTokenSource = new CancellationTokenSource();
792 793 794 795 796 797

		const pendingChanges: IModelContentChangedEvent[] = [];
		const contentChangeListener = this._model.onDidChangeContent((e) => {
			pendingChanges.push(e);
		});

798
		const styling = this._semanticStyling.get(provider);
799

A
Alex Dima 已提交
800 801
		const lastResultId = this._currentDocumentResponse ? this._currentDocumentResponse.resultId || null : null;
		const request = Promise.resolve(provider.provideDocumentSemanticTokens(this._model, lastResultId, this._currentDocumentRequestCancellationTokenSource.token));
A
wip  
Alexandru Dima 已提交
802 803

		request.then((res) => {
A
Alex Dima 已提交
804
			this._currentDocumentRequestCancellationTokenSource = null;
805
			contentChangeListener.dispose();
A
Alex Dima 已提交
806
			this._setDocumentSemanticTokens(provider, res || null, styling, pendingChanges);
A
wip  
Alexandru Dima 已提交
807
		}, (err) => {
808 809 810 811 812 813
			if (!err || typeof err.message !== 'string' || err.message.indexOf('busy') === -1) {
				errors.onUnexpectedError(err);
			}

			// Semantic tokens eats up all errors and considers errors to mean that the result is temporarily not available
			// The API does not have a special error kind to express this...
A
Alex Dima 已提交
814
			this._currentDocumentRequestCancellationTokenSource = null;
815
			contentChangeListener.dispose();
816 817 818

			if (pendingChanges.length > 0) {
				// More changes occurred while the request was running
A
Alex Dima 已提交
819 820
				if (!this._fetchDocumentSemanticTokens.isScheduled()) {
					this._fetchDocumentSemanticTokens.schedule();
821 822
				}
			}
A
wip  
Alexandru Dima 已提交
823 824 825
		});
	}

826 827 828 829 830 831 832 833 834 835 836 837 838 839
	private static _isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens {
		return v && !!((<SemanticTokens>v).data);
	}

	private static _isSemanticTokensEdits(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokensEdits {
		return v && Array.isArray((<SemanticTokensEdits>v).edits);
	}

	private static _copy(src: Uint32Array, srcOffset: number, dest: Uint32Array, destOffset: number, length: number): void {
		for (let i = 0; i < length; i++) {
			dest[destOffset + i] = src[srcOffset + i];
		}
	}

840
	private _setDocumentSemanticTokens(provider: DocumentSemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticTokensProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
A
Alex Dima 已提交
841 842 843 844
		const currentResponse = this._currentDocumentResponse;
		if (this._currentDocumentResponse) {
			this._currentDocumentResponse.dispose();
			this._currentDocumentResponse = null;
A
wip  
Alexandru Dima 已提交
845
		}
846 847
		if (this._isDisposed) {
			// disposed!
848
			if (provider && tokens) {
849
				provider.releaseDocumentSemanticTokens(tokens.resultId);
850 851 852
			}
			return;
		}
853 854 855 856 857 858
		if (!provider || !styling) {
			this._model.setSemanticTokens(null, false);
			return;
		}
		if (!tokens) {
			this._model.setSemanticTokens(null, true);
859 860
			return;
		}
861

862 863 864
		if (ModelSemanticColoring._isSemanticTokensEdits(tokens)) {
			if (!currentResponse) {
				// not possible!
865
				this._model.setSemanticTokens(null, true);
866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899
				return;
			}
			if (tokens.edits.length === 0) {
				// nothing to do!
				tokens = {
					resultId: tokens.resultId,
					data: currentResponse.data
				};
			} else {
				let deltaLength = 0;
				for (const edit of tokens.edits) {
					deltaLength += (edit.data ? edit.data.length : 0) - edit.deleteCount;
				}

				const srcData = currentResponse.data;
				const destData = new Uint32Array(srcData.length + deltaLength);

				let srcLastStart = srcData.length;
				let destLastStart = destData.length;
				for (let i = tokens.edits.length - 1; i >= 0; i--) {
					const edit = tokens.edits[i];

					const copyCount = srcLastStart - (edit.start + edit.deleteCount);
					if (copyCount > 0) {
						ModelSemanticColoring._copy(srcData, srcLastStart - copyCount, destData, destLastStart - copyCount, copyCount);
						destLastStart -= copyCount;
					}

					if (edit.data) {
						ModelSemanticColoring._copy(edit.data, 0, destData, destLastStart - edit.data.length, edit.data.length);
						destLastStart -= edit.data.length;
					}

					srcLastStart = edit.start;
900
				}
901 902 903 904 905 906 907 908 909

				if (srcLastStart > 0) {
					ModelSemanticColoring._copy(srcData, 0, destData, 0, srcLastStart);
				}

				tokens = {
					resultId: tokens.resultId,
					data: destData
				};
A
wip  
Alexandru Dima 已提交
910
			}
911 912 913 914
		}

		if (ModelSemanticColoring._isSemanticTokens(tokens)) {

A
Alex Dima 已提交
915
			this._currentDocumentResponse = new SemanticTokensResponse(provider, tokens.resultId, tokens.data);
916

917
			const result = toMultilineTokens2(tokens, styling, this._model.getLanguageIdentifier());
918 919 920 921 922 923 924 925 926 927 928 929

			// Adjust incoming semantic tokens
			if (pendingChanges.length > 0) {
				// More changes occurred while the request was running
				// We need to:
				// 1. Adjust incoming semantic tokens
				// 2. Request them again
				for (const change of pendingChanges) {
					for (const area of result) {
						for (const singleChange of change.changes) {
							area.applyEdit(singleChange.range, singleChange.text);
						}
930 931
					}
				}
932

A
Alex Dima 已提交
933 934
				if (!this._fetchDocumentSemanticTokens.isScheduled()) {
					this._fetchDocumentSemanticTokens.schedule();
935
				}
936
			}
937

938
			this._model.setSemanticTokens(result, true);
939
			return;
940
		}
941

942
		this._model.setSemanticTokens(null, true);
A
wip  
Alexandru Dima 已提交
943 944
	}

945 946
	private _getSemanticColoringProvider(): DocumentSemanticTokensProvider | null {
		const result = DocumentSemanticTokensProviderRegistry.ordered(this._model);
A
wip  
Alexandru Dima 已提交
947 948
		return (result.length > 0 ? result[0] : null);
	}
A
wip  
Alexandru Dima 已提交
949
}