* Reduces memory consumption of bracket pair colorization by getting rid of PairAstNode._children

* Implements immutable lists that are copied on demand.

src/vs/editor/common/model/bracketPairColorizer/bracketPairColorizer.ts

@@ -149,7 +149,7 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
 					toLength(r.toLineNumber - r.fromLineNumber + 1, 0)
 				)
 			);
-			this.astWithTokens = this.parseDocumentFromTextBuffer(edits, this.astWithTokens);
+			this.astWithTokens = this.parseDocumentFromTextBuffer(edits, this.astWithTokens, false);
 			if (!this.initialAstWithoutTokens) {
 				this.didChangeDecorationsEmitter.fire();
 			}
@@ -159,16 +159,16 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
 			// There are no token information yet
 			const brackets = this.brackets.getSingleLanguageBracketTokens(this.textModel.getLanguageIdentifier().id);
 			const tokenizer = new FastTokenizer(this.textModel.getValue(), brackets);
-			this.initialAstWithoutTokens = parseDocument(tokenizer, [], undefined);
-			this.astWithTokens = this.initialAstWithoutTokens.clone();
+			this.initialAstWithoutTokens = parseDocument(tokenizer, [], undefined, true);
+			this.astWithTokens = this.initialAstWithoutTokens;
 		} else if (textModel.backgroundTokenizationState === BackgroundTokenizationState.Completed) {
 			// Skip the initial ast, as there is no flickering.
 			// Directly create the tree with token information.
 			this.initialAstWithoutTokens = undefined;
-			this.astWithTokens = this.parseDocumentFromTextBuffer([], undefined);
+			this.astWithTokens = this.parseDocumentFromTextBuffer([], undefined, false);
 		} else if (textModel.backgroundTokenizationState === BackgroundTokenizationState.InProgress) {
-			this.initialAstWithoutTokens = this.parseDocumentFromTextBuffer([], undefined);
-			this.astWithTokens = this.initialAstWithoutTokens.clone();
+			this.initialAstWithoutTokens = this.parseDocumentFromTextBuffer([], undefined, true);
+			this.astWithTokens = this.initialAstWithoutTokens;
 		}
 	}
 
@@ -182,21 +182,21 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
 			);
 		}).reverse();
 
-		this.astWithTokens = this.parseDocumentFromTextBuffer(edits, this.astWithTokens);
+		this.astWithTokens = this.parseDocumentFromTextBuffer(edits, this.astWithTokens, false);
 		if (this.initialAstWithoutTokens) {
-			this.initialAstWithoutTokens = this.parseDocumentFromTextBuffer(edits, this.initialAstWithoutTokens);
+			this.initialAstWithoutTokens = this.parseDocumentFromTextBuffer(edits, this.initialAstWithoutTokens, false);
 		}
 	}
 
 	/**
 	 * @pure (only if isPure = true)
 	*/
-	private parseDocumentFromTextBuffer(edits: TextEditInfo[], previousAst: AstNode | undefined): AstNode {
+	private parseDocumentFromTextBuffer(edits: TextEditInfo[], previousAst: AstNode | undefined, immutable: boolean): AstNode {
 		// Is much faster if `isPure = false`.
 		const isPure = false;
-		const previousAstClone = isPure ? previousAst?.clone() : previousAst;
+		const previousAstClone = isPure ? previousAst?.deepClone() : previousAst;
 		const tokenizer = new TextBufferTokenizer(this.textModel, this.brackets);
-		const result = parseDocument(tokenizer, edits, previousAstClone);
+		const result = parseDocument(tokenizer, edits, previousAstClone, immutable);
 		return result;
 	}
 
@@ -236,10 +236,7 @@ function collectBrackets(node: AstNode, nodeOffsetStart: Length, nodeOffsetEnd:
 	} else if (node.kind === AstNodeKind.UnexpectedClosingBracket) {
 		const range = lengthsToRange(nodeOffsetStart, nodeOffsetEnd);
 		result.push(new BracketInfo(range, level - 1, true));
-	} else {
-		if (node.kind === AstNodeKind.Pair) {
-			level++;
-		}
+	} else if (node.kind === AstNodeKind.List) {
 		for (const child of node.children) {
 			nodeOffsetEnd = lengthAdd(nodeOffsetStart, child.length);
 			if (lengthLessThanEqual(nodeOffsetStart, endOffset) && lengthGreaterThanEqual(nodeOffsetEnd, startOffset)) {
@@ -247,6 +244,35 @@ function collectBrackets(node: AstNode, nodeOffsetStart: Length, nodeOffsetEnd:
 			}
 			nodeOffsetStart = nodeOffsetEnd;
 		}
+	} else if (node.kind === AstNodeKind.Pair) {
+		// Don't use node.children here to improve performance
+		level++;
+
+		{
+			const child = node.openingBracket;
+			nodeOffsetEnd = lengthAdd(nodeOffsetStart, child.length);
+			if (lengthLessThanEqual(nodeOffsetStart, endOffset) && lengthGreaterThanEqual(nodeOffsetEnd, startOffset)) {
+				collectBrackets(child, nodeOffsetStart, nodeOffsetEnd, startOffset, endOffset, result, level);
+			}
+			nodeOffsetStart = nodeOffsetEnd;
+		}
+
+		if (node.child) {
+			const child = node.child;
+			nodeOffsetEnd = lengthAdd(nodeOffsetStart, child.length);
+			if (lengthLessThanEqual(nodeOffsetStart, endOffset) && lengthGreaterThanEqual(nodeOffsetEnd, startOffset)) {
+				collectBrackets(child, nodeOffsetStart, nodeOffsetEnd, startOffset, endOffset, result, level);
+			}
+			nodeOffsetStart = nodeOffsetEnd;
+		}
+		if (node.closingBracket) {
+			const child = node.closingBracket;
+			nodeOffsetEnd = lengthAdd(nodeOffsetStart, child.length);
+			if (lengthLessThanEqual(nodeOffsetStart, endOffset) && lengthGreaterThanEqual(nodeOffsetEnd, startOffset)) {
+				collectBrackets(child, nodeOffsetStart, nodeOffsetEnd, startOffset, endOffset, result, level);
+			}
+			nodeOffsetStart = nodeOffsetEnd;
+		}
 	}
 }
 

src/vs/editor/common/model/bracketPairColorizer/concat23Trees.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { AstNode, ListAstNode } from './ast';
+import { AstNode, AstNodeKind, ListAstNode } from './ast';
 
 /**
  * Concatenates a list of (2,3) AstNode's into a single (2,3) AstNode.
@@ -20,6 +20,9 @@ export function concat23Trees(items: AstNode[]): AstNode | null {
 	}
 
 	let i = 0;
+	/**
+	 * Reads nodes of same height and concatenates them to a single node.
+	*/
 	function readNode(): AstNode | null {
 		if (i >= items.length) {
 			return null;
@@ -33,7 +36,7 @@ export function concat23Trees(items: AstNode[]): AstNode | null {
 		}
 
 		if (i - start >= 2) {
-			return concat23TreesOfSameHeight(start === 0 && i === items.length ? items : items.slice(start, i));
+			return concat23TreesOfSameHeight(start === 0 && i === items.length ? items : items.slice(start, i), false);
 		} else {
 			return items[start];
 		}
@@ -61,7 +64,7 @@ export function concat23Trees(items: AstNode[]): AstNode | null {
 	return result;
 }
 
-export function concat23TreesOfSameHeight(items: AstNode[]): AstNode | null {
+export function concat23TreesOfSameHeight(items: AstNode[], createImmutableLists: boolean = false): AstNode | null {
 	if (items.length === 0) {
 		return null;
 	}
@@ -73,16 +76,13 @@ export function concat23TreesOfSameHeight(items: AstNode[]): AstNode | null {
 	// All trees have same height, just create parent nodes.
 	while (length > 3) {
 		const newLength = length >> 1;
-		// Ideally, due to the slice, not a lot of memory is wasted.
-		const newItems = new Array<AstNode>(newLength);
 		for (let i = 0; i < newLength; i++) {
 			const j = i << 1;
-			newItems[i] = ListAstNode.create(items.slice(j, (j + 3 === length) ? length : j + 2));
+			items[i] = ListAstNode.create23(items[j], items[j + 1], j + 3 === length ? items[j + 2] : null, createImmutableLists);
 		}
 		length = newLength;
-		items = newItems;
 	}
-	return ListAstNode.create(items);
+	return ListAstNode.create23(items[0], items[1], length >= 3 ? items[2] : null, createImmutableLists);
 }
 
 function heightDiff(node1: AstNode, node2: AstNode): number {
@@ -91,12 +91,106 @@ function heightDiff(node1: AstNode, node2: AstNode): number {
 
 function concat(node1: AstNode, node2: AstNode): AstNode {
 	if (node1.listHeight === node2.listHeight) {
-		return ListAstNode.create([node1, node2]);
+		return ListAstNode.create23(node1, node2, null, false);
 	}
 	else if (node1.listHeight > node2.listHeight) {
 		// node1 is the tree we want to insert into
-		return (node1 as ListAstNode).append(node2);
+		return append(node1 as ListAstNode, node2);
 	} else {
-		return (node2 as ListAstNode).prepend(node1);
+		return prepend(node2 as ListAstNode, node1);
+	}
+}
+
+/**
+ * Appends the given node to the end of this (2,3) tree.
+ * Returns the new root.
+*/
+function append(list: ListAstNode, nodeToAppend: AstNode): AstNode {
+	list = list.toMutable() as ListAstNode;
+	let curNode: AstNode = list;
+	const parents = new Array<ListAstNode>();
+	let nodeToAppendOfCorrectHeight: AstNode | undefined;
+	while (true) {
+		// assert nodeToInsert.listHeight <= curNode.listHeight
+		if (nodeToAppend.listHeight === curNode.listHeight) {
+			nodeToAppendOfCorrectHeight = nodeToAppend;
+			break;
+		}
+		// assert 0 <= nodeToInsert.listHeight < curNode.listHeight
+		if (curNode.kind !== AstNodeKind.List) {
+			throw new Error('unexpected');
+		}
+		parents.push(curNode);
+		// assert 2 <= curNode.childrenLength <= 3
+		curNode = curNode.makeLastElementMutable()!;
+	}
+	// assert nodeToAppendOfCorrectHeight!.listHeight === curNode.listHeight
+	for (let i = parents.length - 1; i >= 0; i--) {
+		const parent = parents[i];
+		if (nodeToAppendOfCorrectHeight) {
+			// Can we take the element?
+			if (parent.childrenLength >= 3) {
+				// assert parent.childrenLength === 3 && parent.listHeight === nodeToAppendOfCorrectHeight.listHeight + 1
+
+				// we need to split to maintain (2,3)-tree property.
+				// Send the third element + the new element to the parent.
+				nodeToAppendOfCorrectHeight = ListAstNode.create23(parent.unappendChild()!, nodeToAppendOfCorrectHeight, null, false);
+			} else {
+				parent.appendChildOfSameHeight(nodeToAppendOfCorrectHeight);
+				nodeToAppendOfCorrectHeight = undefined;
+			}
+		} else {
+			parent.handleChildrenChanged();
+		}
+	}
+	if (nodeToAppendOfCorrectHeight) {
+		return ListAstNode.create23(list, nodeToAppendOfCorrectHeight, null, false);
+	} else {
+		return list;
+	}
+}
+
+/**
+ * Prepends the given node to the end of this (2,3) tree.
+ * Returns the new root.
+*/
+function prepend(list: ListAstNode, nodeToAppend: AstNode): AstNode {
+	list = list.toMutable() as ListAstNode;
+	let curNode: AstNode = list;
+	const parents = new Array<ListAstNode>();
+	// assert nodeToInsert.listHeight <= curNode.listHeight
+	while (nodeToAppend.listHeight !== curNode.listHeight) {
+		// assert 0 <= nodeToInsert.listHeight < curNode.listHeight
+		if (curNode.kind !== AstNodeKind.List) {
+			throw new Error('unexpected');
+		}
+		parents.push(curNode);
+		// assert 2 <= curNode.childrenFast.length <= 3
+		curNode = curNode.makeFirstElementMutable()!;
+	}
+	let nodeToPrependOfCorrectHeight: AstNode | undefined = nodeToAppend;
+	// assert nodeToAppendOfCorrectHeight!.listHeight === curNode.listHeight
+	for (let i = parents.length - 1; i >= 0; i--) {
+		const parent = parents[i];
+		if (nodeToPrependOfCorrectHeight) {
+			// Can we take the element?
+			if (parent.childrenLength >= 3) {
+				// assert parent.childrenLength === 3 && parent.listHeight === nodeToAppendOfCorrectHeight.listHeight + 1
+
+				// we need to split to maintain (2,3)-tree property.
+				// Send the third element + the new element to the parent.
+				nodeToPrependOfCorrectHeight = ListAstNode.create23(nodeToPrependOfCorrectHeight, parent.unprependChild()!, null, false);
+			} else {
+				parent.prependChildOfSameHeight(nodeToPrependOfCorrectHeight);
+				nodeToPrependOfCorrectHeight = undefined;
+			}
+		} else {
+			parent.handleChildrenChanged();
+		}
+	}
+	if (nodeToPrependOfCorrectHeight) {
+		return ListAstNode.create23(nodeToPrependOfCorrectHeight, list, null, false);
+	} else {
+		return list;
 	}
 }

src/vs/editor/common/model/bracketPairColorizer/length.ts

@@ -9,7 +9,7 @@ import { Range } from 'vs/editor/common/core/range';
 
 /**
  * Represents a non-negative length in terms of line and column count.
- * Prefer using {@link Length}.
+ * Prefer using {@link Length} for performance reasons.
 */
 export class LengthObj {
 	public static zero = new LengthObj(0, 0);

src/vs/editor/common/model/bracketPairColorizer/nodeReader.ts

@@ -24,7 +24,6 @@ export class NodeReader {
 
 	/**
 	 * Returns the longest node at `offset` that satisfies the predicate.
-	 * Has runtime O(log n) where n is the number of nodes in the tree.
 	 * @param offset must be greater than or equal to the last offset this method has been called with!
 	*/
 	readLongestNodeAt(offset: Length, predicate: (node: AstNode) => boolean): AstNode | undefined {
@@ -55,11 +54,12 @@ export class NodeReader {
 					this.nextNodeAfterCurrent();
 				} else {
 					// The reader is somewhere in the current node.
-					if (curNode.children.length > 0) {
+					const nextChildIdx = getNextChildIdx(curNode);
+					if (nextChildIdx !== -1) {
 						// Go to the first child and repeat.
-						this.nextNodes.push(curNode.children[0]);
+						this.nextNodes.push(curNode.getChild(nextChildIdx)!);
 						this.offsets.push(curNodeOffset);
-						this.idxs.push(0);
+						this.idxs.push(nextChildIdx);
 					} else {
 						// We don't have children
 						this.nextNodeAfterCurrent();
@@ -71,16 +71,17 @@ export class NodeReader {
 					this.nextNodeAfterCurrent();
 					return curNode;
 				} else {
+					const nextChildIdx = getNextChildIdx(curNode);
 					// look for shorter node
-					if (curNode.children.length === 0) {
+					if (nextChildIdx === -1) {
 						// There is no shorter node.
 						this.nextNodeAfterCurrent();
 						return undefined;
 					} else {
 						// Descend into first child & repeat.
-						this.nextNodes.push(curNode.children[0]);
+						this.nextNodes.push(curNode.getChild(nextChildIdx)!);
 						this.offsets.push(curNodeOffset);
-						this.idxs.push(0);
+						this.idxs.push(nextChildIdx);
 					}
 				}
 			}
@@ -102,13 +103,12 @@ export class NodeReader {
 
 			// Parent is not undefined, because idxs is not empty
 			const parent = lastOrUndefined(this.nextNodes)!;
+			const nextChildIdx = getNextChildIdx(parent, this.idxs[this.idxs.length - 1]);
 
-			this.idxs[this.idxs.length - 1]++;
-			const parentIdx = this.idxs[this.idxs.length - 1];
-
-			if (parentIdx < parent.children.length) {
-				this.nextNodes.push(parent.children[parentIdx]);
+			if (nextChildIdx !== -1) {
+				this.nextNodes.push(parent.getChild(nextChildIdx)!);
 				this.offsets.push(lengthAdd(currentOffset!, currentNode!.length));
+				this.idxs[this.idxs.length - 1] = nextChildIdx;
 				break;
 			} else {
 				this.idxs.pop();
@@ -119,6 +119,18 @@ export class NodeReader {
 	}
 }
 
+function getNextChildIdx(node: AstNode, curIdx: number = -1): number | -1 {
+	while (true) {
+		curIdx++;
+		if (curIdx >= node.childrenLength) {
+			return -1;
+		}
+		if (node.getChild(curIdx)) {
+			return curIdx;
+		}
+	}
+}
+
 function lastOrUndefined<T>(arr: readonly T[]): T | undefined {
 	return arr.length > 0 ? arr[arr.length - 1] : undefined;
 }

src/vs/editor/common/model/bracketPairColorizer/parser.ts

@@ -11,11 +11,17 @@ import { concat23Trees, concat23TreesOfSameHeight } from './concat23Trees';
 import { NodeReader } from './nodeReader';
 import { OpeningBracketId, Tokenizer, TokenKind } from './tokenizer';
 
-export function parseDocument(tokenizer: Tokenizer, edits: TextEditInfo[], oldNode: AstNode | undefined): AstNode {
-	const parser = new Parser(tokenizer, edits, oldNode);
+/**
+ * Non incrementally built ASTs are immutable.
+*/
+export function parseDocument(tokenizer: Tokenizer, edits: TextEditInfo[], oldNode: AstNode | undefined, createImmutableLists: boolean): AstNode {
+	const parser = new Parser(tokenizer, edits, oldNode, createImmutableLists);
 	return parser.parseDocument();
 }
 
+/**
+ * Non incrementally built ASTs are immutable.
+*/
 class Parser {
 	private readonly oldNodeReader?: NodeReader;
 	private readonly positionMapper: BeforeEditPositionMapper;
@@ -40,7 +46,12 @@ class Parser {
 		private readonly tokenizer: Tokenizer,
 		edits: TextEditInfo[],
 		oldNode: AstNode | undefined,
+		private readonly createImmutableLists: boolean,
 	) {
+		if (oldNode && createImmutableLists) {
+			throw new Error('Not supported');
+		}
+
 		this.oldNodeReader = oldNode ? new NodeReader(oldNode) : undefined;
 		this.positionMapper = new BeforeEditPositionMapper(edits, tokenizer.length);
 	}
@@ -51,7 +62,7 @@ class Parser {
 
 		let result = this.parseList(SmallImmutableSet.getEmpty());
 		if (!result) {
-			result = ListAstNode.create([]);
+			result = ListAstNode.getEmpty();
 		}
 
 		return result;
@@ -73,7 +84,7 @@ class Parser {
 			}
 
 			const child = this.parseChild(openedBracketIds);
-			if (child.kind === AstNodeKind.List && child.children.length === 0) {
+			if (child.kind === AstNodeKind.List && child.childrenLength === 0) {
 				continue;
 			}
 
@@ -81,7 +92,7 @@ class Parser {
 		}
 
 		// When there is no oldNodeReader, all items are created from scratch and must have the same height.
-		const result = this.oldNodeReader ? concat23Trees(items) : concat23TreesOfSameHeight(items);
+		const result = this.oldNodeReader ? concat23Trees(items) : concat23TreesOfSameHeight(items, this.createImmutableLists);
 		return result;
 	}
 

src/vs/editor/test/common/model/bracketPairColorizer/concat23Trees.test.ts

@@ -15,7 +15,7 @@ suite('Bracket Pair Colorizer - mergeItems', () => {
 			new TextAstNode(toLength(1, 1)),
 		]);
 
-		assert.ok(equals(tree, tree.clone()));
+		assert.ok(equals(tree, tree.deepClone()));
 	});
 
 	function equals(node1: AstNode, node2: AstNode): boolean {
@@ -33,7 +33,7 @@ suite('Bracket Pair Colorizer - mergeItems', () => {
 			}
 		}
 
-		if (!node1.missingBracketIds.equals(node2.missingBracketIds)) {
+		if (!node1.missingOpeningBracketIds.equals(node2.missingOpeningBracketIds)) {
 			return false;
 		}
 
@@ -47,7 +47,7 @@ suite('Bracket Pair Colorizer - mergeItems', () => {
 	}
 
 	function testMerge(lists: AstNode[]) {
-		const node = (concat23Trees(lists.map(l => l.clone())) || ListAstNode.create([])).flattenLists();
+		const node = (concat23Trees(lists.map(l => l.deepClone())) || ListAstNode.create([])).flattenLists();
 		// This trivial merge does not maintain the (2,3) tree invariant.
 		const referenceNode = ListAstNode.create(lists).flattenLists();
 
@@ -60,30 +60,30 @@ suite('Bracket Pair Colorizer - mergeItems', () => {
 
 	test('Same Height Lists', () => {
 		const textNode = new TextAstNode(toLength(1, 1));
-		const tree = ListAstNode.create([textNode.clone(), textNode.clone()]);
-		testMerge([tree.clone(), tree.clone(), tree.clone(), tree.clone(), tree.clone()]);
+		const tree = ListAstNode.create([textNode.deepClone(), textNode.deepClone()]);
+		testMerge([tree.deepClone(), tree.deepClone(), tree.deepClone(), tree.deepClone(), tree.deepClone()]);
 	});
 
 	test('Different Height Lists 1', () => {
 		const textNode = new TextAstNode(toLength(1, 1));
-		const tree1 = ListAstNode.create([textNode.clone(), textNode.clone()]);
-		const tree2 = ListAstNode.create([tree1.clone(), tree1.clone()]);
+		const tree1 = ListAstNode.create([textNode.deepClone(), textNode.deepClone()]);
+		const tree2 = ListAstNode.create([tree1.deepClone(), tree1.deepClone()]);
 
 		testMerge([tree1, tree2]);
 	});
 
 	test('Different Height Lists 2', () => {
 		const textNode = new TextAstNode(toLength(1, 1));
-		const tree1 = ListAstNode.create([textNode.clone(), textNode.clone()]);
-		const tree2 = ListAstNode.create([tree1.clone(), tree1.clone()]);
+		const tree1 = ListAstNode.create([textNode.deepClone(), textNode.deepClone()]);
+		const tree2 = ListAstNode.create([tree1.deepClone(), tree1.deepClone()]);
 
 		testMerge([tree2, tree1]);
 	});
 
 	test('Different Height Lists 3', () => {
 		const textNode = new TextAstNode(toLength(1, 1));
-		const tree1 = ListAstNode.create([textNode.clone(), textNode.clone()]);
-		const tree2 = ListAstNode.create([tree1.clone(), tree1.clone()]);
+		const tree1 = ListAstNode.create([textNode.deepClone(), textNode.deepClone()]);
+		const tree2 = ListAstNode.create([tree1.deepClone(), tree1.deepClone()]);
 
 		testMerge([tree2, tree1, tree1, tree2, tree2]);
 	});