safe point

src/vs/editor/common/model/indentationGuesser.ts

@@ -6,6 +6,8 @@
 
 import { CharCode } from 'vs/base/common/charCode';
 import { TextBuffer } from 'vs/editor/common/model/textBuffer';
+import { TextBuffer as TextBuffer2 } from 'vs/editor/common/model/textBuffer2';
+import { IRawPTBuffer } from './textSource';
 
 export interface IIndentationGuesserTarget {
 	getLineCount(): number;
@@ -15,7 +17,7 @@ export interface IIndentationGuesserTarget {
 export class IndentationGuesserTextBufferTarget implements IIndentationGuesserTarget {
 
 	constructor(
-		private readonly _buffer: TextBuffer
+		private readonly _buffer: TextBuffer | TextBuffer2
 	) { }
 
 	public getLineCount(): number {
@@ -42,6 +44,27 @@ export class IndentationGuesserStringArrayTarget implements IIndentationGuesserT
 	}
 }
 
+export class IndentationGuesserRawTextBufferTarget implements IIndentationGuesserTarget {
+
+	constructor(
+		private readonly _rawBuffer: IRawPTBuffer
+	) { }
+
+	public getLineCount(): number {
+		return this._rawBuffer.length;
+	}
+
+	public getLineContent(lineNumber: number): string {
+		if (lineNumber === 1) {
+			return this._rawBuffer.text.substring(0, this._rawBuffer.lineStarts[0]);
+		} else if (lineNumber === this._rawBuffer.lineStarts.length + 1) {
+			return this._rawBuffer.text.substring(this._rawBuffer.lineStarts[this._rawBuffer.lineStarts.length - 1] + 1);
+		}
+
+		return this._rawBuffer.text.substring(this._rawBuffer.lineStarts[lineNumber - 2] + 1, this._rawBuffer.lineStarts[lineNumber - 1]);
+	}
+}
+
 /**
  * Compute the diff in spaces between two line's indentation.
  */

src/vs/editor/common/model/modelLine.ts

@@ -13,6 +13,7 @@ import { Range } from 'vs/editor/common/core/range';
 import { IModelTokensChangedEvent } from 'vs/editor/common/model/textModelEvents';
 import { onUnexpectedError } from 'vs/base/common/errors';
 import { TextBuffer } from 'vs/editor/common/model/textBuffer';
+import { TextBuffer as TextBuffer2 } from 'vs/editor/common/model/textBuffer2';
 import { TokenizationResult2 } from 'vs/editor/common/core/token';
 import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 
@@ -253,7 +254,7 @@ export class ModelLinesTokens {
 		return (firstInvalidLineNumber >= lineNumber);
 	}
 
-	public hasLinesToTokenize(buffer: TextBuffer): boolean {
+	public hasLinesToTokenize(buffer: TextBuffer | TextBuffer2): boolean {
 		return (this._invalidLineStartIndex < buffer.getLineCount());
 	}
 
@@ -419,7 +420,7 @@ export class ModelLinesTokens {
 
 	//#region Tokenization
 
-	public _tokenizeOneLine(buffer: TextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number {
+	public _tokenizeOneLine(buffer: TextBuffer | TextBuffer2, eventBuilder: ModelTokensChangedEventBuilder): number {
 		if (!this.hasLinesToTokenize(buffer)) {
 			return buffer.getLineCount() + 1;
 		}
@@ -428,7 +429,7 @@ export class ModelLinesTokens {
 		return lineNumber;
 	}
 
-	public _updateTokensUntilLine(buffer: TextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
+	public _updateTokensUntilLine(buffer: TextBuffer | TextBuffer2, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
 		if (!this.tokenizationSupport) {
 			this._invalidLineStartIndex = buffer.getLineCount();
 			return;

src/vs/editor/common/model/textBuffer.ts

@@ -55,7 +55,7 @@ export class TextBuffer {
 	private _lineStarts: PrefixSumComputer;
 
 	constructor(textSource: ITextSource) {
-		this._lines = textSource.lines.slice(0);
+		this._lines = (<string[]>textSource.lines).slice(0);
 		this._BOM = textSource.BOM;
 		this._EOL = textSource.EOL;
 		this._mightContainRTL = textSource.containsRTL;

src/vs/editor/common/model/textModel.ts

@@ -10,13 +10,14 @@ import { Position, IPosition } from 'vs/editor/common/core/position';
 import { Range, IRange } from 'vs/editor/common/core/range';
 import { Selection } from 'vs/editor/common/core/selection';
 import * as editorCommon from 'vs/editor/common/editorCommon';
-import { guessIndentation, IndentationGuesserTextBufferTarget, IndentationGuesserStringArrayTarget } from 'vs/editor/common/model/indentationGuesser';
+import { guessIndentation, IndentationGuesserTextBufferTarget, IndentationGuesserStringArrayTarget, IndentationGuesserRawTextBufferTarget } from 'vs/editor/common/model/indentationGuesser';
 import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions';
 import { TextModelSearch, SearchParams } from 'vs/editor/common/model/textModelSearch';
 import { TextSource, ITextSource, IRawTextSource, RawTextSource } from 'vs/editor/common/model/textSource';
 import { IModelContentChangedEvent, ModelRawContentChangedEvent, ModelRawFlush, ModelRawEOLChanged, IModelOptionsChangedEvent, InternalModelContentChangeEvent } from 'vs/editor/common/model/textModelEvents';
 import { Disposable, IDisposable } from 'vs/base/common/lifecycle';
-import { TextBuffer } from 'vs/editor/common/model/textBuffer';
+// import { TextBuffer } from 'vs/editor/common/model/textBuffer';
+import { TextBuffer } from 'vs/editor/common/model/textBuffer2';
 
 const LIMIT_FIND_COUNT = 999;
 export const LONG_LINE_BOUNDARY = 10000;
@@ -48,7 +49,13 @@ export class TextModel extends Disposable implements editorCommon.ITextModel {
 
 		let resolvedOpts: editorCommon.TextModelResolvedOptions;
 		if (options.detectIndentation) {
-			const guessedIndentation = guessIndentation(new IndentationGuesserStringArrayTarget(textSource.lines), options.tabSize, options.insertSpaces);
+			const guessedIndentation = guessIndentation(
+				Array.isArray(textSource.lines) ?
+					new IndentationGuesserStringArrayTarget(textSource.lines) :
+					new IndentationGuesserRawTextBufferTarget(textSource.lines),
+				options.tabSize,
+				options.insertSpaces
+			);
 			resolvedOpts = new editorCommon.TextModelResolvedOptions({
 				tabSize: guessedIndentation.tabSize,
 				insertSpaces: guessedIndentation.insertSpaces,
@@ -412,7 +419,7 @@ export class TextModel extends Disposable implements editorCommon.ITextModel {
 
 	public getLinesContent(): string[] {
 		this._assertNotDisposed();
-		return this._buffer.getLinesContent();
+		return this._buffer.getLinesContent().split(/\r\n|\r|\n/);
 	}
 
 	public getEOL(): string {

src/vs/editor/common/model/textSource.ts

@@ -7,6 +7,18 @@
 import * as strings from 'vs/base/common/strings';
 import { DefaultEndOfLine } from 'vs/editor/common/editorCommon';
 
+/**
+ * Raw text buffer for Piece Table.
+ */
+export interface IRawPTBuffer {
+	text: string;
+	lineStarts: number[];
+	/**
+	 * lines count
+	 */
+	length: number;
+}
+
 /**
  * A processed string ready to be turned into an editor model.
  */
@@ -18,7 +30,7 @@ export interface IRawTextSource {
 	/**
 	 * The text split into lines.
 	 */
-	readonly lines: string[];
+	readonly lines: string[] | IRawPTBuffer;
 	/**
 	 * The BOM (leading character sequence of the file).
 	 */
@@ -51,18 +63,28 @@ export class RawTextSource {
 		const isBasicASCII = (containsRTL ? false : strings.isBasicASCII(rawText));
 
 		// Split the text into lines
-		const lines = rawText.split(/\r\n|\r|\n/);
+		// const lines = rawText.split(/\r\n|\r|\n/);
 
 		// Remove the BOM (if present)
 		let BOM = '';
-		if (strings.startsWithUTF8BOM(lines[0])) {
+		if (strings.startsWithUTF8BOM(rawText)) {
 			BOM = strings.UTF8_BOM_CHARACTER;
-			lines[0] = lines[0].substr(1);
+			rawText = rawText.substr(1);
+		}
+
+		let lastLineFeed = -1;
+		var lineStarts = [];
+		while ((lastLineFeed = rawText.indexOf('\n', lastLineFeed + 1)) !== -1) {
+			lineStarts.push(lastLineFeed + 1);
 		}
 
 		return {
 			BOM: BOM,
-			lines: lines,
+			lines: {
+				text: rawText,
+				lineStarts: lineStarts,
+				length: lineStarts.length
+			},
 			length: rawText.length,
 			containsRTL: containsRTL,
 			isBasicASCII: isBasicASCII,
@@ -83,7 +105,7 @@ export interface ITextSource {
 	/**
 	 * The text split into lines.
 	 */
-	readonly lines: string[];
+	readonly lines: string[] | IRawPTBuffer;
 	/**
 	 * The BOM (leading character sequence of the file).
 	 */

src/vs/editor/common/services/modelServiceImpl.ts

@@ -409,10 +409,25 @@ export class ModelServiceImpl implements IModelService {
 		};
 		const textSourceLineSequence = new class implements ISequence {
 			public getLength(): number {
-				return textSource.lines.length;
+				return textSource.lines.length + 1;
 			}
 			public getElementHash(index: number): string {
-				return textSource.lines[index];
+				if (Array.isArray(textSource.lines)) {
+					return textSource.lines[index];
+				} else {
+					if (textSource.lines.length === 0) {
+						return textSource.lines.text;
+					}
+					if (index === 0) {
+						return textSource.lines.text.substring(0, textSource.lines.lineStarts[0]);
+					}
+
+					if (index === textSource.lines.length) {
+						return textSource.lines.text.substring(textSource.lines.lineStarts[index - 1]);
+					}
+
+					return textSource.lines.text.substring(textSource.lines.lineStarts[index - 1], textSource.lines.lineStarts[index]);
+				}
 			}
 		};
 

src/vs/editor/common/viewModel/prefixSumComputer.ts

@@ -23,7 +23,7 @@ export class PrefixSumComputer {
 	/**
 	 * values[i] is the value at index i
 	 */
-	private values: Uint32Array;
+	values: Uint32Array;
 
 	/**
 	 * prefixSum[i] = SUM(heights[j]), 0 <= j <= i

src/vs/workbench/common/editor/textEditorModel.ts

@@ -106,8 +106,10 @@ export abstract class BaseTextEditorModel extends EditorModel implements ITextEd
 			}
 
 			return firstLineText.substr(0, Math.min(crIndex, lfIndex));
-		} else {
+		} else if (Array.isArray(value.lines)) {
 			return value.lines[0].substr(0, 100);
+		} else {
+			return value.lines.text.substr(0, 100);
 		}
 	}
 

src/vs/workbench/parts/welcome/walkThrough/node/walkThroughContentProvider.ts

@@ -81,7 +81,7 @@ export class WalkThroughSnippetContentProvider implements ITextModelContentProvi
 					return '';
 				};
 
-				const markdown = content.value.lines.join('\n');
+				const markdown = Array.isArray(content.value.lines) ? content.value.lines.join('\n') : content.value.lines.text;
 				marked(markdown, { renderer });
 
 				const modeId = this.modeService.getModeIdForLanguageName(languageName);

src/vs/workbench/services/backup/node/backupFileService.ts

@@ -215,7 +215,7 @@ export class BackupFileService implements IBackupFileService {
 
 	public parseBackupContent(rawTextSource: IRawTextSource): string {
 		const textSource = TextSource.fromRawTextSource(rawTextSource, DefaultEndOfLine.LF);
-		return textSource.lines.slice(1).join(textSource.EOL); // The first line of a backup text file is the file name
+		return (<string[]>textSource.lines).slice(1).join(textSource.EOL); // The first line of a backup text file is the file name
 	}
 
 	public toBackupResource(resource: Uri): Uri {

src/vs/workbench/services/textfile/electron-browser/modelBuilder.ts

@@ -56,6 +56,61 @@ function optimizeStringMemory(buff: Buffer, s: string): string {
 	return s;
 }
 
+class PTBasedBuilder {
+	private lineStarts: number[];
+	private text: string;
+	private BOM: string;
+	private chunkIndex: number;
+	private totalLength: number;
+
+	constructor() {
+		this.text = '';
+		this.BOM = '';
+		this.chunkIndex = 0;
+		this.totalLength = 0;
+		this.lineStarts = [];
+	}
+
+	public acceptChunk(chunk: string): void {
+		if (this.chunkIndex === 0) {
+			if (strings.startsWithUTF8BOM(chunk)) {
+				this.BOM = strings.UTF8_BOM_CHARACTER;
+				chunk = chunk.substr(1);
+			}
+		}
+
+		let lastLineFeed = -1;
+		while ((lastLineFeed = chunk.indexOf('\n', lastLineFeed + 1)) !== -1) {
+			this.lineStarts.push(this.totalLength + lastLineFeed + 1);
+		}
+
+		this.text += chunk;
+		this.chunkIndex++;
+	}
+
+	public finish(containsRTL: boolean, isBasicASCII: boolean): ModelBuilderResult {
+		if (this.lineStarts[this.lineStarts.length - 1] > this.totalLength) {
+			this.lineStarts.pop();
+		}
+
+		return {
+			hash: null,
+			value: {
+				length: this.totalLength,
+				lines: {
+					text: this.text,
+					lineStarts: this.lineStarts,
+					length: this.lineStarts.length
+				},
+				BOM: this.BOM,
+				totalCRCount: 0,
+				containsRTL: containsRTL,
+				isBasicASCII: isBasicASCII
+			}
+		};
+	}
+}
+
 class ModelLineBasedBuilder {
 
 	private computeHash: boolean;
@@ -126,13 +181,15 @@ export class ModelBuilder {
 	private containsRTL: boolean;
 	private isBasicASCII: boolean;
 
+	private ptBasedBuilder: PTBasedBuilder;
+
 	public static fromStringStream(stream: IStringStream): TPromise<ModelBuilderResult> {
 		return new TPromise<ModelBuilderResult>((c, e, p) => {
 			let done = false;
 			let builder = new ModelBuilder(false);
 
 			stream.on('data', (chunk) => {
-				builder.acceptChunk(chunk);
+				builder.acceptChunk2(chunk);
 			});
 
 			stream.on('error', (error) => {
@@ -145,7 +202,7 @@ export class ModelBuilder {
 			stream.on('end', () => {
 				if (!done) {
 					done = true;
-					c(builder.finish());
+					c(builder.finish2());
 				}
 			});
 		});
@@ -159,6 +216,7 @@ export class ModelBuilder {
 		this.totalLength = 0;
 		this.containsRTL = false;
 		this.isBasicASCII = true;
+		this.ptBasedBuilder = new PTBasedBuilder();
 	}
 
 	private _updateCRCount(chunk: string): void {
@@ -210,6 +268,22 @@ export class ModelBuilder {
 		this.leftoverPrevChunk = lines[lines.length - 1];
 	}
 
+	public acceptChunk2(chunk: string): void {
+		if (chunk.length === 0) {
+			return;
+		}
+
+		// update lineStart to offset mapping
+		if (!this.containsRTL) {
+			this.containsRTL = strings.containsRTL(chunk);
+		}
+		if (this.isBasicASCII) {
+			this.isBasicASCII = strings.isBasicASCII(chunk);
+		}
+
+		this.ptBasedBuilder.acceptChunk(chunk);
+	}
+
 	public finish(): ModelBuilderResult {
 		let finalLines = [this.leftoverPrevChunk];
 		if (this.leftoverEndsInCR) {
@@ -218,4 +292,8 @@ export class ModelBuilder {
 		this.lineBasedBuilder.acceptLines(finalLines);
 		return this.lineBasedBuilder.finish(this.totalLength, this.totalCRCount, this.containsRTL, this.isBasicASCII);
 	}
+
+	public finish2(): ModelBuilderResult {
+		return this.ptBasedBuilder.finish(this.containsRTL, this.isBasicASCII);
+	}
 }

src/vs/workbench/test/workbenchTestServices.ts

@@ -859,7 +859,7 @@ export class TestBackupFileService implements IBackupFileService {
 	}
 
 	public parseBackupContent(rawText: IRawTextSource): string {
-		return rawText.lines.join('\n');
+		return Array.isArray(rawText.lines) ? rawText.lines.join('\n') : rawText.lines.text;
 	}
 
 	public discardResourceBackup(resource: URI): TPromise<void> {