diff --git a/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase.ts b/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase.ts index e5cc6a1b2912d3728bb27fca38d7351ff86e13ec..7b38ac8baecba6f85a2904009e685de073c2d11a 100644 --- a/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase.ts +++ b/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase.ts @@ -13,6 +13,7 @@ import { SearchData, isValidMatch, Searcher, createFindMatch } from 'vs/editor/c import { FindMatch } from 'vs/editor/common/model'; // const lfRegex = new RegExp(/\r\n|\r|\n/g); +export const AverageBufferSize = 65535; export function createUintArray(arr: number[]): Uint32Array | Uint16Array { let r; @@ -311,7 +312,7 @@ export class PieceTreeBase { } normalizeEOL(eol: '\r\n' | '\n') { - let averageBufferSize = 65536; + let averageBufferSize = AverageBufferSize; let min = averageBufferSize - Math.floor(averageBufferSize / 3); let max = min * 2; @@ -712,7 +713,8 @@ export class PieceTreeBase { if (node.piece.bufferIndex === 0 && piece.end.line === this._lastChangeBufferPos.line && piece.end.column === this._lastChangeBufferPos.column && - (nodeStartOffset + piece.length === offset) + (nodeStartOffset + piece.length === offset) && + value.length < AverageBufferSize ) { // changed buffer this.appendToNode(node, value); @@ -769,19 +771,27 @@ export class PieceTreeBase { this.deleteNodeTail(node, insertPosInBuffer); } - let newPiece = this.createNewPiece(value); + let newPieces = this.createNewPieces(value); if (newRightPiece.length > 0) { this.rbInsertRight(node, newRightPiece); } - this.rbInsertRight(node, newPiece); + + let tmpNode = node; + for (let k = 0; k < newPieces.length; k++) { + tmpNode = this.rbInsertRight(tmpNode, newPieces[k]); + } this.deleteNodes(nodesToDel); } else { this.insertContentToNodeRight(value, node); } } else { // insert new node - let piece = this.createNewPiece(value); - this.rbInsertLeft(null, piece); + let pieces = this.createNewPieces(value); + let node = this.rbInsertLeft(null, pieces[0]); + + for (let k = 1; k < pieces.length; k++) { + node = this.rbInsertRight(node, pieces[k]); + } } // todo, this is too brutal. Total line feed count should be updated the same way as lf_left. @@ -887,8 +897,11 @@ export class PieceTreeBase { } } - let newPiece = this.createNewPiece(value); - let newNode = this.rbInsertLeft(node, newPiece); + let newPieces = this.createNewPieces(value); + let newNode = this.rbInsertLeft(node, newPieces[newPieces.length - 1]); + for (let k = newPieces.length - 2; k >= 0; k--) { + newNode = this.rbInsertLeft(newNode, newPieces[k]); + } this.validateCRLFWithPrevNode(newNode); this.deleteNodes(nodesToDel); } @@ -900,8 +913,14 @@ export class PieceTreeBase { value += '\n'; } - let newPiece = this.createNewPiece(value); - let newNode = this.rbInsertRight(node, newPiece); + let newPieces = this.createNewPieces(value); + let newNode = this.rbInsertRight(node, newPieces[0]); + let tmpNode = newNode; + + for (let k = 1; k < newPieces.length; k++) { + tmpNode = this.rbInsertRight(tmpNode, newPieces[k]); + } + this.validateCRLFWithPrevNode(newNode); } @@ -994,7 +1013,47 @@ export class PieceTreeBase { } } - createNewPiece(text: string): Piece { + createNewPieces(text: string): Piece[] { + if (text.length > AverageBufferSize) { + // the content is large, operations like substring, charCode becomes slow + // so here we split it into smaller chunks, just like what we did for CR/LF normalization + let newPieces = []; + while (text.length > AverageBufferSize) { + const lastChar = text.charCodeAt(AverageBufferSize - 1); + let splitText; + if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) { + // last character is \r or a high surrogate => keep it back + splitText = text.substring(0, AverageBufferSize - 1); + text = text.substring(AverageBufferSize - 1); + } else { + splitText = text.substring(0, AverageBufferSize); + text = text.substring(AverageBufferSize); + } + + let lineStarts = createLineStartsFast(splitText); + newPieces.push(new Piece( + this._buffers.length, /* buffer index */ + { line: 0, column: 0 }, + { line: lineStarts.length - 1, column: splitText.length - lineStarts[lineStarts.length - 1] }, + lineStarts.length - 1, + splitText.length + )); + this._buffers.push(new StringBuffer(splitText, lineStarts)); + } + + let lineStarts = createLineStartsFast(text); + newPieces.push(new Piece( + this._buffers.length, /* buffer index */ + { line: 0, column: 0 }, + { line: lineStarts.length - 1, column: text.length - lineStarts[lineStarts.length - 1] }, + lineStarts.length - 1, + text.length + )); + this._buffers.push(new StringBuffer(text, lineStarts)); + + return newPieces; + } + let startOffset = this._buffers[0].buffer.length; const lineStarts = createLineStartsFast(text, false); @@ -1029,14 +1088,14 @@ export class PieceTreeBase { let endColumn = endOffset - this._buffers[0].lineStarts[endIndex]; let endPos = { line: endIndex, column: endColumn }; let newPiece = new Piece( - 0, + 0, /** todo */ start, endPos, this.getLineFeedCnt(0, start, endPos), endOffset - startOffset ); this._lastChangeBufferPos = endPos; - return newPiece; + return [newPiece]; } getLinesRawContent(): string { @@ -1519,8 +1578,8 @@ export class PieceTreeBase { } // create new piece which contains \r\n - let piece = this.createNewPiece('\r\n'); - this.rbInsertRight(prev, piece); + let pieces = this.createNewPieces('\r\n'); + this.rbInsertRight(prev, pieces[0]); // delete empty nodes for (let i = 0; i < nodesToDel.length; i++) { diff --git a/src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts b/src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts index 266e9de61ae4ed006d7a7ca25834bcc52f9b44d5..717870f0b59fd7d204e7be72d0ad43490dfd9aef 100644 --- a/src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts +++ b/src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts @@ -1445,11 +1445,44 @@ suite('centralized lineStarts with CRLF', () => { }); suite('random is unsupervised', () => { + test('splitting large change buffer', function () { + let pieceTable = createTextBuffer([''], false); + let str = ''; + + pieceTable.insert(0, 'WUZ\nXVZY\n'); + str = str.substring(0, 0) + 'WUZ\nXVZY\n' + str.substring(0); + pieceTable.insert(8, '\r\r\nZXUWVW'); + str = str.substring(0, 8) + '\r\r\nZXUWVW' + str.substring(8); + pieceTable.delete(10, 7); + str = str.substring(0, 10) + str.substring(10 + 7); + pieceTable.delete(10, 1); + str = str.substring(0, 10) + str.substring(10 + 1); + pieceTable.insert(4, 'VX\r\r\nWZVZ'); + str = str.substring(0, 4) + 'VX\r\r\nWZVZ' + str.substring(4); + pieceTable.delete(11, 3); + str = str.substring(0, 11) + str.substring(11 + 3); + pieceTable.delete(12, 4); + str = str.substring(0, 12) + str.substring(12 + 4); + pieceTable.delete(8, 0); + str = str.substring(0, 8) + str.substring(8 + 0); + pieceTable.delete(10, 2); + str = str.substring(0, 10) + str.substring(10 + 2); + pieceTable.insert(0, 'VZXXZYZX\r'); + str = str.substring(0, 0) + 'VZXXZYZX\r' + str.substring(0); + + assert.equal(pieceTable.getLinesRawContent(), str); + + testLineStarts(str, pieceTable); + testLinesContent(str, pieceTable); + assertTreeInvariants(pieceTable); + }); + test('random insert delete', function () { this.timeout(500000); let str = ''; let pieceTable = createTextBuffer([str], false); + // let output = ''; for (let i = 0; i < 1000; i++) { if (Math.random() < 0.6) { // insert @@ -1457,6 +1490,8 @@ suite('random is unsupervised', () => { let pos = randomInt(str.length + 1); pieceTable.insert(pos, text); str = str.substring(0, pos) + text + str.substring(pos); + // output += `pieceTable.insert(${pos}, '${text.replace(/\n/g, '\\n').replace(/\r/g, '\\r')}');\n`; + // output += `str = str.substring(0, ${pos}) + '${text.replace(/\n/g, '\\n').replace(/\r/g, '\\r')}' + str.substring(${pos});\n`; } else { // delete let pos = randomInt(str.length); @@ -1466,8 +1501,12 @@ suite('random is unsupervised', () => { ); pieceTable.delete(pos, length); str = str.substring(0, pos) + str.substring(pos + length); + // output += `pieceTable.delete(${pos}, ${length});\n`; + // output += `str = str.substring(0, ${pos}) + str.substring(${pos} + ${length});\n` + } } + // console.log(output); assert.equal(pieceTable.getLinesRawContent(), str);