Merge pull request #41172 from Microsoft/rebornix/buffer-pt

buffer improvement with piece tree

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Range } from 'vs/editor/common/core/range';
+import { Position } from 'vs/editor/common/core/position';
+import * as strings from 'vs/base/common/strings';
+import { IValidatedEditOperation } from 'vs/editor/common/model/linesTextBuffer/linesTextBuffer';
+import { PieceTreeBase, StringBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
+import { IIdentifiedSingleEditOperation, EndOfLinePreference, ITextBuffer, ApplyEditsResult, IInternalModelContentChange } from 'vs/editor/common/model';
+import { ITextSnapshot } from 'vs/platform/files/common/files';
+
+export class PieceTreeTextBuffer implements ITextBuffer {
+	private _pieceTree: PieceTreeBase;
+	private _BOM: string;
+	private _EOL: string;
+	private _EOLLength: number;
+	private _mightContainRTL: boolean;
+	private _mightContainNonBasicASCII: boolean;
+
+	constructor(chunks: StringBuffer[], BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, isBasicASCII: boolean) {
+		this._BOM = BOM;
+		this._EOL = eol;
+		this._EOLLength = this._EOL.length;
+		this._mightContainNonBasicASCII = !isBasicASCII;
+		this._mightContainRTL = containsRTL;
+		this._pieceTree = new PieceTreeBase(chunks);
+	}
+
+	// #region TextBuffer
+	public equals(other: ITextBuffer): boolean {
+		if (!(other instanceof PieceTreeTextBuffer)) {
+			return false;
+		}
+		if (this._BOM !== other._BOM) {
+			return false;
+		}
+		if (this._EOL !== other._EOL) {
+			return false;
+		}
+		return this._pieceTree.equal(other._pieceTree);
+	}
+	public mightContainRTL(): boolean {
+		return this._mightContainRTL;
+	}
+	public mightContainNonBasicASCII(): boolean {
+		return this._mightContainNonBasicASCII;
+	}
+	public getBOM(): string {
+		return this._BOM;
+	}
+	public getEOL(): string {
+		return this._EOL;
+	}
+
+	public createSnapshot(preserveBOM: boolean): ITextSnapshot {
+		return this._pieceTree.createSnapshot(preserveBOM ? this._BOM : '');
+	}
+
+	public getOffsetAt(lineNumber: number, column: number): number {
+		return this._pieceTree.getOffsetAt(lineNumber, column);
+	}
+
+	public getPositionAt(offset: number): Position {
+		return this._pieceTree.getPositionAt(offset);
+	}
+
+	public getRangeAt(start: number, length: number): Range {
+		let end = start + length;
+		const startPosition = this.getPositionAt(start);
+		const endPosition = this.getPositionAt(end);
+		return new Range(startPosition.lineNumber, startPosition.column, endPosition.lineNumber, endPosition.column);
+	}
+
+	public getValueInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): string {
+		if (range.isEmpty()) {
+			return '';
+		}
+
+		const lineEnding = this._getEndOfLine(eol);
+		const text = this._pieceTree.getValueInRange(range);
+		return text.replace(/\r\n|\r|\n/g, lineEnding);
+	}
+
+	public getValueLengthInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): number {
+		if (range.isEmpty()) {
+			return 0;
+		}
+
+		if (range.startLineNumber === range.endLineNumber) {
+			return (range.endColumn - range.startColumn);
+		}
+
+		let startOffset = this.getOffsetAt(range.startLineNumber, range.startColumn);
+		let endOffset = this.getOffsetAt(range.endLineNumber, range.endColumn);
+		return endOffset - startOffset;
+	}
+
+	public getLength(): number {
+		return this._pieceTree.getLength();
+	}
+
+	public getLineCount(): number {
+		return this._pieceTree.getLineCount();
+	}
+
+	public getLinesContent(): string[] {
+		return this._pieceTree.getLinesContent();
+	}
+
+	public getLineContent(lineNumber: number): string {
+		return this._pieceTree.getLineContent(lineNumber);
+	}
+
+	public getLineCharCode(lineNumber: number, index: number): number {
+		return this._pieceTree.getLineCharCode(lineNumber, index);
+	}
+
+	public getLineLength(lineNumber: number): number {
+		if (lineNumber === this.getLineCount()) {
+			let startOffset = this.getOffsetAt(lineNumber, 1);
+			return this.getLength() - startOffset;
+		}
+		return this.getOffsetAt(lineNumber + 1, 1) - this.getOffsetAt(lineNumber, 1) - this._EOLLength;
+	}
+
+	public getLineMinColumn(lineNumber: number): number {
+		return 1;
+	}
+
+	public getLineMaxColumn(lineNumber: number): number {
+		return this.getLineLength(lineNumber) + 1;
+	}
+
+	public getLineFirstNonWhitespaceColumn(lineNumber: number): number {
+		const result = strings.firstNonWhitespaceIndex(this.getLineContent(lineNumber));
+		if (result === -1) {
+			return 0;
+		}
+		return result + 1;
+	}
+
+	public getLineLastNonWhitespaceColumn(lineNumber: number): number {
+		const result = strings.lastNonWhitespaceIndex(this.getLineContent(lineNumber));
+		if (result === -1) {
+			return 0;
+		}
+		return result + 2;
+	}
+
+	private _getEndOfLine(eol: EndOfLinePreference): string {
+		switch (eol) {
+			case EndOfLinePreference.LF:
+				return '\n';
+			case EndOfLinePreference.CRLF:
+				return '\r\n';
+			case EndOfLinePreference.TextDefined:
+				return this.getEOL();
+		}
+		throw new Error('Unknown EOL preference');
+	}
+
+	public setEOL(newEOL: '\r\n' | '\n'): void {
+		this._EOL = newEOL;
+		this._EOLLength = this._EOL.length;
+		this._pieceTree.normalizeEOL(newEOL);
+	}
+
+	public applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult {
+		let mightContainRTL = this._mightContainRTL;
+		let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
+		let canReduceOperations = true;
+
+		let operations: IValidatedEditOperation[] = [];
+		for (let i = 0; i < rawOperations.length; i++) {
+			let op = rawOperations[i];
+			if (canReduceOperations && op._isTracked) {
+				canReduceOperations = false;
+			}
+			let validatedRange = op.range;
+			if (!mightContainRTL && op.text) {
+				// check if the new inserted text contains RTL
+				mightContainRTL = strings.containsRTL(op.text);
+			}
+			if (!mightContainNonBasicASCII && op.text) {
+				mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+			}
+			operations[i] = {
+				sortIndex: i,
+				identifier: op.identifier,
+				range: validatedRange,
+				rangeOffset: this.getOffsetAt(validatedRange.startLineNumber, validatedRange.startColumn),
+				rangeLength: this.getValueLengthInRange(validatedRange),
+				lines: op.text ? op.text.split(/\r\n|\r|\n/) : null,
+				forceMoveMarkers: op.forceMoveMarkers,
+				isAutoWhitespaceEdit: op.isAutoWhitespaceEdit || false
+			};
+		}
+
+		// Sort operations ascending
+		operations.sort(PieceTreeTextBuffer._sortOpsAscending);
+
+		for (let i = 0, count = operations.length - 1; i < count; i++) {
+			let rangeEnd = operations[i].range.getEndPosition();
+			let nextRangeStart = operations[i + 1].range.getStartPosition();
+
+			if (nextRangeStart.isBefore(rangeEnd)) {
+				// overlapping ranges
+				throw new Error('Overlapping ranges are not allowed!');
+			}
+		}
+
+		if (canReduceOperations) {
+			operations = this._reduceOperations(operations);
+		}
+
+		// Delta encode operations
+		let reverseRanges = PieceTreeTextBuffer._getInverseEditRanges(operations);
+		let newTrimAutoWhitespaceCandidates: { lineNumber: number, oldContent: string }[] = [];
+
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			if (recordTrimAutoWhitespace && op.isAutoWhitespaceEdit && op.range.isEmpty()) {
+				// Record already the future line numbers that might be auto whitespace removal candidates on next edit
+				for (let lineNumber = reverseRange.startLineNumber; lineNumber <= reverseRange.endLineNumber; lineNumber++) {
+					let currentLineContent = '';
+					if (lineNumber === reverseRange.startLineNumber) {
+						currentLineContent = this.getLineContent(op.range.startLineNumber);
+						if (strings.firstNonWhitespaceIndex(currentLineContent) !== -1) {
+							continue;
+						}
+					}
+					newTrimAutoWhitespaceCandidates.push({ lineNumber: lineNumber, oldContent: currentLineContent });
+				}
+			}
+		}
+
+		let reverseOperations: IIdentifiedSingleEditOperation[] = [];
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			reverseOperations[i] = {
+				identifier: op.identifier,
+				range: reverseRange,
+				text: this.getValueInRange(op.range),
+				forceMoveMarkers: op.forceMoveMarkers
+			};
+		}
+
+		this._mightContainRTL = mightContainRTL;
+		this._mightContainNonBasicASCII = mightContainNonBasicASCII;
+
+		const contentChanges = this._doApplyEdits(operations);
+
+		let trimAutoWhitespaceLineNumbers: number[] = null;
+		if (recordTrimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) {
+			// sort line numbers auto whitespace removal candidates for next edit descending
+			newTrimAutoWhitespaceCandidates.sort((a, b) => b.lineNumber - a.lineNumber);
+
+			trimAutoWhitespaceLineNumbers = [];
+			for (let i = 0, len = newTrimAutoWhitespaceCandidates.length; i < len; i++) {
+				let lineNumber = newTrimAutoWhitespaceCandidates[i].lineNumber;
+				if (i > 0 && newTrimAutoWhitespaceCandidates[i - 1].lineNumber === lineNumber) {
+					// Do not have the same line number twice
+					continue;
+				}
+
+				let prevContent = newTrimAutoWhitespaceCandidates[i].oldContent;
+				let lineContent = this.getLineContent(lineNumber);
+
+				if (lineContent.length === 0 || lineContent === prevContent || strings.firstNonWhitespaceIndex(lineContent) !== -1) {
+					continue;
+				}
+
+				trimAutoWhitespaceLineNumbers.push(lineNumber);
+			}
+		}
+
+		return new ApplyEditsResult(
+			reverseOperations,
+			contentChanges,
+			trimAutoWhitespaceLineNumbers
+		);
+	}
+
+	/**
+ * Transform operations such that they represent the same logic edit,
+ * but that they also do not cause OOM crashes.
+ */
+	private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
+		if (operations.length < 1000) {
+			// We know from empirical testing that a thousand edits work fine regardless of their shape.
+			return operations;
+		}
+
+		// At one point, due to how events are emitted and how each operation is handled,
+		// some operations can trigger a high ammount of temporary string allocations,
+		// that will immediately get edited again.
+		// e.g. a formatter inserting ridiculous ammounts of \n on a model with a single line
+		// Therefore, the strategy is to collapse all the operations into a huge single edit operation
+		return [this._toSingleEditOperation(operations)];
+	}
+
+	_toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation {
+		let forceMoveMarkers = false,
+			firstEditRange = operations[0].range,
+			lastEditRange = operations[operations.length - 1].range,
+			entireEditRange = new Range(firstEditRange.startLineNumber, firstEditRange.startColumn, lastEditRange.endLineNumber, lastEditRange.endColumn),
+			lastEndLineNumber = firstEditRange.startLineNumber,
+			lastEndColumn = firstEditRange.startColumn,
+			result: string[] = [];
+
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let operation = operations[i],
+				range = operation.range;
+
+			forceMoveMarkers = forceMoveMarkers || operation.forceMoveMarkers;
+
+			// (1) -- Push old text
+			for (let lineNumber = lastEndLineNumber; lineNumber < range.startLineNumber; lineNumber++) {
+				if (lineNumber === lastEndLineNumber) {
+					result.push(this.getLineContent(lineNumber).substring(lastEndColumn - 1));
+				} else {
+					result.push('\n');
+					result.push(this.getLineContent(lineNumber));
+				}
+			}
+
+			if (range.startLineNumber === lastEndLineNumber) {
+				result.push(this.getLineContent(range.startLineNumber).substring(lastEndColumn - 1, range.startColumn - 1));
+			} else {
+				result.push('\n');
+				result.push(this.getLineContent(range.startLineNumber).substring(0, range.startColumn - 1));
+			}
+
+			// (2) -- Push new text
+			if (operation.lines) {
+				for (let j = 0, lenJ = operation.lines.length; j < lenJ; j++) {
+					if (j !== 0) {
+						result.push('\n');
+					}
+					result.push(operation.lines[j]);
+				}
+			}
+
+			lastEndLineNumber = operation.range.endLineNumber;
+			lastEndColumn = operation.range.endColumn;
+		}
+
+		return {
+			sortIndex: 0,
+			identifier: operations[0].identifier,
+			range: entireEditRange,
+			rangeOffset: this.getOffsetAt(entireEditRange.startLineNumber, entireEditRange.startColumn),
+			rangeLength: this.getValueLengthInRange(entireEditRange, EndOfLinePreference.TextDefined),
+			lines: result.join('').split('\n'),
+			forceMoveMarkers: forceMoveMarkers,
+			isAutoWhitespaceEdit: false
+		};
+	}
+
+	private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] {
+		operations.sort(PieceTreeTextBuffer._sortOpsDescending);
+
+		let contentChanges: IInternalModelContentChange[] = [];
+
+		// operations are from bottom to top
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+
+			const startLineNumber = op.range.startLineNumber;
+			const startColumn = op.range.startColumn;
+			const endLineNumber = op.range.endLineNumber;
+			const endColumn = op.range.endColumn;
+
+			if (startLineNumber === endLineNumber && startColumn === endColumn && (!op.lines || op.lines.length === 0)) {
+				// no-op
+				continue;
+			}
+
+			const deletingLinesCnt = endLineNumber - startLineNumber;
+			const insertingLinesCnt = (op.lines ? op.lines.length - 1 : 0);
+			const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
+
+			const text = (op.lines ? op.lines.join(this.getEOL()) : '');
+
+			if (text) {
+				// replacement
+				this._pieceTree.delete(op.rangeOffset, op.rangeLength);
+				this._pieceTree.insert(op.rangeOffset, text);
+
+			} else {
+				// deletion
+				this._pieceTree.delete(op.rangeOffset, op.rangeLength);
+			}
+
+			if (editingLinesCnt < insertingLinesCnt) {
+				let newLinesContent: string[] = [];
+				for (let j = editingLinesCnt + 1; j <= insertingLinesCnt; j++) {
+					newLinesContent.push(op.lines[j]);
+				}
+
+				newLinesContent[newLinesContent.length - 1] = this.getLineContent(startLineNumber + insertingLinesCnt - 1);
+			}
+
+			const contentChangeRange = new Range(startLineNumber, startColumn, endLineNumber, endColumn);
+			contentChanges.push({
+				range: contentChangeRange,
+				rangeLength: op.rangeLength,
+				text: text,
+				lines: op.lines,
+				rangeOffset: op.rangeOffset,
+				forceMoveMarkers: op.forceMoveMarkers
+			});
+		}
+		return contentChanges;
+	}
+
+	// #endregion
+
+	// #region helper
+	// testing purpose.
+	public getPieceTree(): PieceTreeBase {
+		return this._pieceTree;
+	}
+	/**
+	 * Assumes `operations` are validated and sorted ascending
+	 */
+	public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] {
+		let result: Range[] = [];
+
+		let prevOpEndLineNumber: number;
+		let prevOpEndColumn: number;
+		let prevOp: IValidatedEditOperation = null;
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let op = operations[i];
+
+			let startLineNumber: number;
+			let startColumn: number;
+
+			if (prevOp) {
+				if (prevOp.range.endLineNumber === op.range.startLineNumber) {
+					startLineNumber = prevOpEndLineNumber;
+					startColumn = prevOpEndColumn + (op.range.startColumn - prevOp.range.endColumn);
+				} else {
+					startLineNumber = prevOpEndLineNumber + (op.range.startLineNumber - prevOp.range.endLineNumber);
+					startColumn = op.range.startColumn;
+				}
+			} else {
+				startLineNumber = op.range.startLineNumber;
+				startColumn = op.range.startColumn;
+			}
+
+			let resultRange: Range;
+
+			if (op.lines && op.lines.length > 0) {
+				// the operation inserts something
+				let lineCount = op.lines.length;
+				let firstLine = op.lines[0];
+				let lastLine = op.lines[lineCount - 1];
+
+				if (lineCount === 1) {
+					// single line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn + firstLine.length);
+				} else {
+					// multi line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber + lineCount - 1, lastLine.length + 1);
+				}
+			} else {
+				// There is nothing to insert
+				resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn);
+			}
+
+			prevOpEndLineNumber = resultRange.endLineNumber;
+			prevOpEndColumn = resultRange.endColumn;
+
+			result.push(resultRange);
+			prevOp = op;
+		}
+
+		return result;
+	}
+
+	private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return a.sortIndex - b.sortIndex;
+		}
+		return r;
+	}
+
+	private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return b.sortIndex - a.sortIndex;
+		}
+		return -r;
+	}
+	// #endregion
+}

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { ITextBufferBuilder, DefaultEndOfLine, ITextBufferFactory, ITextBuffer } from 'vs/editor/common/model';
+import { PieceTreeTextBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer';
+import { StringBuffer, createLineStarts, createLineStartsFast } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
+import { CharCode } from 'vs/base/common/charCode';
+
+export class PieceTreeTextBufferFactory implements ITextBufferFactory {
+
+	constructor(
+		private readonly _chunks: StringBuffer[],
+		private readonly _bom: string,
+		private readonly _cr: number,
+		private readonly _lf: number,
+		private readonly _crlf: number,
+		private readonly _containsRTL: boolean,
+		private readonly _isBasicASCII: boolean,
+		private readonly _normalizeEOL: boolean
+	) { }
+
+	private _getEOL(defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
+		const totalEOLCount = this._cr + this._lf + this._crlf;
+		const totalCRCount = this._cr + this._crlf;
+		if (totalEOLCount === 0) {
+			// This is an empty file or a file with precisely one line
+			return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
+		}
+		if (totalCRCount > totalEOLCount / 2) {
+			// More than half of the file contains \r\n ending lines
+			return '\r\n';
+		}
+		// At least one line more ends in \n
+		return '\n';
+	}
+
+	public create(defaultEOL: DefaultEndOfLine): ITextBuffer {
+		const eol = this._getEOL(defaultEOL);
+		let chunks = this._chunks;
+
+		if (this._normalizeEOL &&
+			((eol === '\r\n' && (this._cr > 0 || this._lf > 0))
+				|| (eol === '\n' && (this._cr > 0 || this._crlf > 0)))
+		) {
+			// Normalize pieces
+			for (let i = 0, len = chunks.length; i < len; i++) {
+				let str = chunks[i].buffer.replace(/\r\n|\r|\n/g, eol);
+				let newLineStart = createLineStartsFast(str);
+				chunks[i] = new StringBuffer(str, newLineStart);
+			}
+		}
+
+		return new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._isBasicASCII);
+	}
+
+	public getFirstLineText(lengthLimit: number): string {
+		return this._chunks[0].buffer.substr(0, 100).split(/\r\n|\r|\n/)[0];
+	}
+}
+
+export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
+	private chunks: StringBuffer[];
+	private BOM: string;
+
+	private _hasPreviousChar: boolean;
+	private _previousChar: number;
+	private _tmpLineStarts: number[];
+
+	private cr: number;
+	private lf: number;
+	private crlf: number;
+	private containsRTL: boolean;
+	private isBasicASCII: boolean;
+
+	constructor() {
+		this.chunks = [];
+		this.BOM = '';
+
+		this._hasPreviousChar = false;
+		this._previousChar = 0;
+		this._tmpLineStarts = [];
+
+		this.cr = 0;
+		this.lf = 0;
+		this.crlf = 0;
+		this.containsRTL = false;
+		this.isBasicASCII = true;
+	}
+
+	public acceptChunk(chunk: string): void {
+		if (chunk.length === 0) {
+			return;
+		}
+
+		if (this.chunks.length === 0) {
+			if (strings.startsWithUTF8BOM(chunk)) {
+				this.BOM = strings.UTF8_BOM_CHARACTER;
+				chunk = chunk.substr(1);
+			}
+		}
+
+		const lastChar = chunk.charCodeAt(chunk.length - 1);
+		if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) {
+			// last character is \r or a high surrogate => keep it back
+			this._acceptChunk1(chunk.substr(0, chunk.length - 1), false);
+			this._hasPreviousChar = true;
+			this._previousChar = lastChar;
+		} else {
+			this._acceptChunk1(chunk, false);
+			this._hasPreviousChar = false;
+			this._previousChar = lastChar;
+		}
+	}
+
+	private _acceptChunk1(chunk: string, allowEmptyStrings: boolean): void {
+		if (!allowEmptyStrings && chunk.length === 0) {
+			// Nothing to do
+			return;
+		}
+
+		if (this._hasPreviousChar) {
+			this._acceptChunk2(String.fromCharCode(this._previousChar) + chunk);
+		} else {
+			this._acceptChunk2(chunk);
+		}
+	}
+
+	private _acceptChunk2(chunk: string): void {
+		const lineStarts = createLineStarts(this._tmpLineStarts, chunk);
+
+		this.chunks.push(new StringBuffer(chunk, lineStarts.lineStarts));
+		this.cr += lineStarts.cr;
+		this.lf += lineStarts.lf;
+		this.crlf += lineStarts.crlf;
+
+		if (this.isBasicASCII) {
+			this.isBasicASCII = lineStarts.isBasicASCII;
+		}
+		if (!this.isBasicASCII && !this.containsRTL) {
+			// No need to check if is basic ASCII
+			this.containsRTL = strings.containsRTL(chunk);
+		}
+	}
+
+	public finish(normalizeEOL: boolean = true): PieceTreeTextBufferFactory {
+		this._finish();
+		return new PieceTreeTextBufferFactory(
+			this.chunks,
+			this.BOM,
+			this.cr,
+			this.lf,
+			this.crlf,
+			this.containsRTL,
+			this.isBasicASCII,
+			normalizeEOL
+		);
+	}
+
+	private _finish(): void {
+		if (this.chunks.length === 0) {
+			this._acceptChunk1('', true);
+		}
+
+		if (this._hasPreviousChar) {
+			this._hasPreviousChar = false;
+			// recreate last chunk
+			let lastChunk = this.chunks[this.chunks.length - 1];
+			lastChunk.buffer += String.fromCharCode(this._previousChar);
+			let newLineStarts = createLineStartsFast(lastChunk.buffer);
+			lastChunk.lineStarts = newLineStarts;
+			if (this._previousChar === CharCode.CarriageReturn) {
+				this.cr++;
+			}
+		}
+	}
+}

src/vs/editor/common/model/textModel.ts

@@ -34,11 +34,17 @@ import { TextModelSearch, SearchParams } from 'vs/editor/common/model/textModelS
 import { TPromise } from 'vs/base/common/winjs.base';
 import { IStringStream, ITextSnapshot } from 'vs/platform/files/common/files';
 import { LinesTextBufferBuilder } from 'vs/editor/common/model/linesTextBuffer/linesTextBufferBuilder';
+import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
 import { ChunksTextBufferBuilder } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder';
 
 // Here is the master switch for the text buffer implementation:
+const USE_PIECE_TREE_IMPLEMENTATION = false;
 const USE_CHUNKS_TEXT_BUFFER = false;
+
 function createTextBufferBuilder() {
+	if (USE_PIECE_TREE_IMPLEMENTATION) {
+		return new PieceTreeTextBufferBuilder();
+	}
 	if (USE_CHUNKS_TEXT_BUFFER) {
 		return new ChunksTextBufferBuilder();
 	}

src/vs/editor/test/common/model/benchmark/benchmarkUtils.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ITextBufferBuilder, ITextBufferFactory, ITextBuffer, DefaultEndOfLine } from 'vs/editor/common/model';
+import { LinesTextBufferBuilder } from 'vs/editor/common/model/linesTextBuffer/linesTextBufferBuilder';
+import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
+import { ChunksTextBufferBuilder } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder';
+
+export function doBenchmark<T>(id: string, ts: T[], fn: (t: T) => void) {
+	let columns: string[] = [id];
+	for (let i = 0; i < ts.length; i++) {
+		var start = process.hrtime();
+		fn(ts[i]);
+		var diff = process.hrtime(start);
+		columns.push(`${(diff[0] * 1000 + diff[1] / 1000000).toFixed(3)} ms`);
+	}
+	console.log('|' + columns.join('\t|') + '|');
+}
+
+export interface IBenchmark {
+	name: string;
+	/**
+	 * Before each cycle, this function will be called to create TextBufferFactory
+	 */
+	buildBuffer: (textBufferBuilder: ITextBufferBuilder) => ITextBufferFactory;
+	/**
+	 * Before each cycle, this function will be called to do pre-work for text buffer.
+	 * This will be called onece `buildBuffer` is finished.
+	 */
+	preCycle: (textBuffer: ITextBuffer) => void;
+	/**
+	 * The function we are benchmarking
+	 */
+	fn: (textBuffer: ITextBuffer) => void;
+}
+
+export class BenchmarkSuite {
+	name: string;
+	iterations: number;
+	benchmarks: IBenchmark[];
+
+	constructor(suiteOptions: { name: string, iterations: number }) {
+		this.name = suiteOptions.name;
+		this.iterations = suiteOptions.iterations;
+		this.benchmarks = [];
+	}
+
+	add(benchmark: IBenchmark) {
+		this.benchmarks.push(benchmark);
+	}
+
+	run() {
+		console.log(`|${this.name}\t|line buffer\t|piece table\t|edcore\t`);
+		console.log('|---|---|---|---|');
+		for (let i = 0; i < this.benchmarks.length; i++) {
+			let benchmark = this.benchmarks[i];
+			let columns: string[] = [benchmark.name];
+			[new LinesTextBufferBuilder(), new PieceTreeTextBufferBuilder(), new ChunksTextBufferBuilder()].forEach((builder: ITextBufferBuilder) => {
+				let timeDiffTotal = 0.0;
+				for (let j = 0; j < this.iterations; j++) {
+					let factory = benchmark.buildBuffer(builder);
+					let buffer = factory.create(DefaultEndOfLine.LF);
+					benchmark.preCycle(buffer);
+					var start = process.hrtime();
+					benchmark.fn(buffer);
+					var diff = process.hrtime(start);
+					timeDiffTotal += (diff[0] * 1000 * 1000 + diff[1] / 1000);
+				}
+				columns.push(`${(timeDiffTotal / 1000 / this.iterations).toFixed(3)} ms`);
+			});
+			console.log('|' + columns.join('\t|') + '|');
+		}
+		console.log('\n');
+	}
+}

src/vs/editor/test/common/model/benchmark/bootstrap.js

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+require('../../../../../../bootstrap-amd').bootstrap('vs/editor/test/common/model/benchmark/entry');
\ No newline at end of file

src/vs/editor/test/common/model/benchmark/entry.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import 'vs/editor/test/common/model/benchmark/modelbuilder.benchmark';
+import 'vs/editor/test/common/model/benchmark/operations.benchmark';
+import 'vs/editor/test/common/model/benchmark/searchNReplace.benchmark';
\ No newline at end of file

src/vs/editor/test/common/model/benchmark/modelbuilder.benchmark.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { LinesTextBufferBuilder } from 'vs/editor/common/model/linesTextBuffer/linesTextBufferBuilder';
+import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
+import { ITextBufferBuilder } from 'vs/editor/common/model';
+import { generateRandomChunkWithLF } from 'vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils';
+import { doBenchmark } from 'vs/editor/test/common/model/benchmark/benchmarkUtils';
+
+let linesTextBufferBuilder = new LinesTextBufferBuilder();
+let pieceTreeTextBufferBuilder = new PieceTreeTextBufferBuilder();
+let chunks = [];
+
+for (let i = 0; i < 100; i++) {
+	chunks.push(generateRandomChunkWithLF(16 * 1000, 64 * 1000));
+}
+
+let modelBuildBenchmark = function (id: string, builders: ITextBufferBuilder[], chunkCnt: number) {
+	doBenchmark(id, builders, builder => {
+		for (let i = 0, len = Math.min(chunkCnt, chunks.length); i < len; i++) {
+			builder.acceptChunk(chunks[i]);
+		}
+		builder.finish();
+	});
+};
+
+console.log(`|model builder\t|line buffer\t|piece table\t|`);
+console.log('|---|---|---|');
+for (let i of [10, 100]) {
+	modelBuildBenchmark(`${i} random chunks`, [linesTextBufferBuilder, pieceTreeTextBufferBuilder], i);
+}

src/vs/editor/test/common/model/benchmark/operations.benchmark.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ITextBufferBuilder, EndOfLinePreference } from 'vs/editor/common/model';
+import { BenchmarkSuite } from 'vs/editor/test/common/model/benchmark/benchmarkUtils';
+import { generateRandomChunkWithLF, generateRandomEdits, generateSequentialInserts, getRandomInt } from 'vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils';
+import { Range } from 'vs/editor/common/core/range';
+
+let fileSizes = [1, 1000, 64 * 1000, 32 * 1000 * 1000];
+let editTypes = [
+	{
+		id: 'random edits',
+		generateEdits: generateRandomEdits
+	},
+	{
+		id: 'sequential inserts',
+		generateEdits: generateSequentialInserts
+	}
+];
+
+for (let fileSize of fileSizes) {
+	let chunks = [];
+
+	let chunkCnt = Math.floor(fileSize / (64 * 1000));
+	if (chunkCnt === 0) {
+		chunks.push(generateRandomChunkWithLF(fileSize, fileSize));
+	} else {
+		let chunk = generateRandomChunkWithLF(64 * 1000, 64 * 1000);
+		// try to avoid OOM
+		for (let j = 0; j < chunkCnt; j++) {
+			chunks.push(Buffer.from(chunk + j).toString());
+		}
+	}
+
+	for (let editType of editTypes) {
+		const edits = editType.generateEdits(chunks, 1000);
+
+		let editsSuite = new BenchmarkSuite({
+			name: `File Size: ${fileSize}Byte, ${editType.id}`,
+			iterations: 10
+		});
+
+		for (let i of [10, 100, 1000]) {
+			editsSuite.add({
+				name: `apply ${i} edits`,
+				buildBuffer: (textBufferBuilder: ITextBufferBuilder) => {
+					chunks.forEach(ck => textBufferBuilder.acceptChunk(ck));
+					return textBufferBuilder.finish();
+				},
+				preCycle: (textBuffer) => {
+					return textBuffer;
+				},
+				fn: (textBuffer) => {
+					// for line model, this loop doesn't reflect the real situation.
+					for (let k = 0; k < edits.length && k < i; k++) {
+						textBuffer.applyEdits([edits[k]], false);
+					}
+				}
+			});
+
+			editsSuite.add({
+				name: `Read all lines after ${i} edits`,
+				buildBuffer: (textBufferBuilder: ITextBufferBuilder) => {
+					chunks.forEach(ck => textBufferBuilder.acceptChunk(ck));
+					return textBufferBuilder.finish();
+				},
+				preCycle: (textBuffer) => {
+					for (let k = 0; k < edits.length && k < i; k++) {
+						textBuffer.applyEdits([edits[k]], false);
+					}
+					return textBuffer;
+				},
+				fn: (textBuffer) => {
+					for (let j = 0, len = textBuffer.getLineCount(); j < len; j++) {
+						var str = textBuffer.getLineContent(j + 1);
+						let firstChar = str.charCodeAt(0);
+						let lastChar = str.charCodeAt(str.length - 1);
+						firstChar = firstChar - lastChar;
+						lastChar = firstChar + lastChar;
+						firstChar = lastChar - firstChar;
+					}
+				}
+			});
+
+			editsSuite.add({
+				name: `Read 10 random windows after ${i} edits`,
+				buildBuffer: (textBufferBuilder: ITextBufferBuilder) => {
+					chunks.forEach(ck => textBufferBuilder.acceptChunk(ck));
+					return textBufferBuilder.finish();
+				},
+				preCycle: (textBuffer) => {
+					for (let k = 0; k < edits.length && k < i; k++) {
+						textBuffer.applyEdits([edits[k]], false);
+					}
+					return textBuffer;
+				},
+				fn: (textBuffer) => {
+					for (let i = 0; i < 10; i++) {
+						let minLine = 1;
+						let maxLine = textBuffer.getLineCount();
+						let startLine = getRandomInt(minLine, Math.max(minLine, maxLine - 100));
+						let endLine = Math.min(maxLine, startLine + 100);
+						for (let j = startLine; j < endLine; j++) {
+							var str = textBuffer.getLineContent(j + 1);
+							let firstChar = str.charCodeAt(0);
+							let lastChar = str.charCodeAt(str.length - 1);
+							firstChar = firstChar - lastChar;
+							lastChar = firstChar + lastChar;
+							firstChar = lastChar - firstChar;
+						}
+					}
+				}
+			});
+
+			editsSuite.add({
+				name: `save file after ${i} edits`,
+				buildBuffer: (textBufferBuilder: ITextBufferBuilder) => {
+					chunks.forEach(ck => textBufferBuilder.acceptChunk(ck));
+					return textBufferBuilder.finish();
+				},
+				preCycle: (textBuffer) => {
+					for (let k = 0; k < edits.length && k < i; k++) {
+						textBuffer.applyEdits([edits[k]], false);
+					}
+					return textBuffer;
+				},
+				fn: (textBuffer) => {
+					const lineCount = textBuffer.getLineCount();
+					const fullModelRange = new Range(1, 1, lineCount, textBuffer.getLineLength(lineCount) + 1);
+					textBuffer.getValueInRange(fullModelRange, EndOfLinePreference.LF);
+				}
+			});
+		}
+
+		editsSuite.run();
+	}
+}
\ No newline at end of file

src/vs/editor/test/common/model/benchmark/searchNReplace.benchmark.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { ITextBufferBuilder } from 'vs/editor/common/model';
+import { generateRandomReplaces, generateRandomChunkWithLF } from 'vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils';
+import { BenchmarkSuite } from 'vs/editor/test/common/model/benchmark/benchmarkUtils';
+
+let fileSizes = [1, 1000, 64 * 1000, 32 * 1000 * 1000];
+
+for (let fileSize of fileSizes) {
+	let chunks = [];
+
+	let chunkCnt = Math.floor(fileSize / (64 * 1000));
+	if (chunkCnt === 0) {
+		chunks.push(generateRandomChunkWithLF(fileSize, fileSize));
+	} else {
+		let chunk = generateRandomChunkWithLF(64 * 1000, 64 * 1000);
+		// try to avoid OOM
+		for (let j = 0; j < chunkCnt; j++) {
+			chunks.push(Buffer.from(chunk + j).toString());
+		}
+	}
+
+	let replaceSuite = new BenchmarkSuite({
+		name: `File Size: ${fileSize}Byte`,
+		iterations: 10
+	});
+
+	let edits = generateRandomReplaces(chunks, 500, 5, 10);
+
+	for (let i of [10, 100, 500]) {
+		replaceSuite.add({
+			name: `replace ${i} occurrences`,
+			buildBuffer: (textBufferBuilder: ITextBufferBuilder) => {
+				chunks.forEach(ck => textBufferBuilder.acceptChunk(ck));
+				return textBufferBuilder.finish();
+			},
+			preCycle: (textBuffer) => {
+				return textBuffer;
+			},
+			fn: (textBuffer) => {
+				textBuffer.applyEdits(edits.slice(0, i), false);
+			}
+		});
+	}
+
+	replaceSuite.run();
+}
\ No newline at end of file

src/vs/editor/test/common/model/linesTextBuffer/linesTextBufferBuilderAuto.test.ts

@@ -5,7 +5,7 @@
 'use strict';
 
 import { testModelBuilder } from './linesTextBufferBuilder.test';
-import { CharCode } from 'vs/base/common/charCode';
+import { getRandomInt, getRandomEOLSequence, getRandomString } from 'vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils';
 
 const GENERATE_TESTS = false;
 
@@ -21,30 +21,6 @@ suite('ModelBuilder Auto Tests', () => {
 
 });
 
-function getRandomInt(min: number, max: number): number {
-	return Math.floor(Math.random() * (max - min + 1)) + min;
-}
-
-function getRandomEOLSequence(): string {
-	let rnd = getRandomInt(1, 3);
-	if (rnd === 1) {
-		return '\n';
-	}
-	if (rnd === 2) {
-		return '\r';
-	}
-	return '\r\n';
-}
-
-function getRandomString(minLength: number, maxLength: number): string {
-	let length = getRandomInt(minLength, maxLength);
-	let r = '';
-	for (let i = 0; i < length; i++) {
-		r += String.fromCharCode(getRandomInt(CharCode.a, CharCode.z));
-	}
-	return r;
-}
-
 function generateRandomFile(): string {
 	let lineCount = getRandomInt(1, 10);
 	let mixedEOLSequence = getRandomInt(1, 2) === 1 ? true : false;

src/vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { CharCode } from 'vs/base/common/charCode';
+import { IIdentifiedSingleEditOperation, DefaultEndOfLine, ITextBufferBuilder, ITextBuffer } from 'vs/editor/common/model';
+import { Range } from 'vs/editor/common/core/range';
+
+export function getRandomInt(min: number, max: number): number {
+	return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+export function getRandomEOLSequence(): string {
+	let rnd = getRandomInt(1, 3);
+	if (rnd === 1) {
+		return '\n';
+	}
+	if (rnd === 2) {
+		return '\r';
+	}
+	return '\r\n';
+}
+
+export function getRandomString(minLength: number, maxLength: number): string {
+	let length = getRandomInt(minLength, maxLength);
+	let r = '';
+	for (let i = 0; i < length; i++) {
+		r += String.fromCharCode(getRandomInt(CharCode.a, CharCode.z));
+	}
+	return r;
+}
+
+export function generateRandomEdits(chunks: string[], editCnt: number): IIdentifiedSingleEditOperation[] {
+	let lines = [];
+	for (let i = 0; i < chunks.length; i++) {
+		let newLines = chunks[i].split(/\r\n|\r|\n/);
+		if (lines.length === 0) {
+			lines.push(...newLines);
+		} else {
+			newLines[0] = lines[lines.length - 1] + newLines[0];
+			lines.splice(lines.length - 1, 1, ...newLines);
+		}
+	}
+
+	let ops: IIdentifiedSingleEditOperation[] = [];
+
+	for (let i = 0; i < editCnt; i++) {
+		let line = getRandomInt(1, lines.length);
+		let startColumn = getRandomInt(1, Math.max(lines[line - 1].length, 1));
+		let endColumn = getRandomInt(startColumn, Math.max(lines[line - 1].length, startColumn));
+		let text: string = '';
+		if (Math.random() < .5) {
+			text = getRandomString(5, 10);
+		}
+
+		ops.push({
+			text: text,
+			range: new Range(line, startColumn, line, endColumn)
+		});
+		lines[line - 1] = lines[line - 1].substring(0, startColumn - 1) + text + lines[line - 1].substring(endColumn - 1);
+	}
+
+	return ops;
+}
+
+export function generateSequentialInserts(chunks: string[], editCnt: number): IIdentifiedSingleEditOperation[] {
+	let lines = [];
+	for (let i = 0; i < chunks.length; i++) {
+		let newLines = chunks[i].split(/\r\n|\r|\n/);
+		if (lines.length === 0) {
+			lines.push(...newLines);
+		} else {
+			newLines[0] = lines[lines.length - 1] + newLines[0];
+			lines.splice(lines.length - 1, 1, ...newLines);
+		}
+	}
+
+	let ops: IIdentifiedSingleEditOperation[] = [];
+
+	for (let i = 0; i < editCnt; i++) {
+		let line = lines.length;
+		let column = lines[line - 1].length + 1;
+		let text: string = '';
+		if (Math.random() < .5) {
+			text = '\n';
+			lines.push('');
+		} else {
+			text = getRandomString(1, 2);
+			lines[line - 1] += text;
+		}
+
+		ops.push({
+			text: text,
+			range: new Range(line, column, line, column)
+		});
+	}
+
+	return ops;
+}
+
+export function generateRandomReplaces(chunks: string[], editCnt: number, searchStringLen: number, replaceStringLen: number): IIdentifiedSingleEditOperation[] {
+	let lines = [];
+	for (let i = 0; i < chunks.length; i++) {
+		let newLines = chunks[i].split(/\r\n|\r|\n/);
+		if (lines.length === 0) {
+			lines.push(...newLines);
+		} else {
+			newLines[0] = lines[lines.length - 1] + newLines[0];
+			lines.splice(lines.length - 1, 1, ...newLines);
+		}
+	}
+
+	let ops: IIdentifiedSingleEditOperation[] = [];
+	let chunkSize = Math.max(1, Math.floor(lines.length / editCnt));
+	let chunkCnt = Math.floor(lines.length / chunkSize);
+	let replaceString = getRandomString(replaceStringLen, replaceStringLen);
+
+	let previousChunksLength = 0;
+	for (let i = 0; i < chunkCnt; i++) {
+		let startLine = previousChunksLength + 1;
+		let endLine = previousChunksLength + chunkSize;
+		let line = getRandomInt(startLine, endLine);
+		let maxColumn = lines[line - 1].length + 1;
+		let startColumn = getRandomInt(1, maxColumn);
+		let endColumn = Math.min(maxColumn, startColumn + searchStringLen);
+
+		ops.push({
+			text: replaceString,
+			range: new Range(line, startColumn, line, endColumn)
+		});
+		previousChunksLength = endLine;
+	}
+
+	return ops;
+}
+
+export function createMockText(lineCount: number, minColumn: number, maxColumn: number) {
+	let fixedEOL = getRandomEOLSequence();
+	let lines: string[] = [];
+	for (let i = 0; i < lineCount; i++) {
+		if (i !== 0) {
+			lines.push(fixedEOL);
+		}
+		lines.push(getRandomString(minColumn, maxColumn));
+	}
+	return lines.join('');
+}
+
+export function createMockBuffer(str: string, bufferBuilder: ITextBufferBuilder): ITextBuffer {
+	bufferBuilder.acceptChunk(str);
+	let bufferFactory = bufferBuilder.finish();
+	let buffer = bufferFactory.create(DefaultEndOfLine.LF);
+	return buffer;
+}
+
+export function generateRandomChunkWithLF(minLength: number, maxLength: number): string {
+	let length = getRandomInt(minLength, maxLength);
+	let r = '';
+	for (let i = 0; i < length; i++) {
+		let randomI = getRandomInt(0, CharCode.z - CharCode.a + 1);
+		if (randomI === 0 && Math.random() < 0.3) {
+			r += '\n';
+		} else {
+			r += String.fromCharCode(randomI + CharCode.a - 1);
+		}
+	}
+	return r;
+}
\ No newline at end of file