diff --git a/src/vs/editor/common/editorCommon.ts b/src/vs/editor/common/editorCommon.ts index 83039a5e2927365bc62fa8f5fbd3eb769103bd4c..1d4f2820aad20255b0901d4236c33f7099453bd4 100644 --- a/src/vs/editor/common/editorCommon.ts +++ b/src/vs/editor/common/editorCommon.ts @@ -1171,16 +1171,13 @@ export interface ITokensInflatorMap { export interface ILineTokensBinaryEncoding { START_INDEX_MASK: number; TYPE_MASK: number; - BRACKET_MASK: number; START_INDEX_OFFSET: number; TYPE_OFFSET: number; - BRACKET_OFFSET: number; deflateArr(map:ITokensInflatorMap, tokens:Modes.IToken[]): number[]; inflate(map:ITokensInflatorMap, binaryEncodedToken:number): Modes.IToken; getStartIndex(binaryEncodedToken:number): number; getType(map:ITokensInflatorMap, binaryEncodedToken:number): string; - getBracket(binaryEncodedToken:number): Modes.Bracket; inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[]): Modes.IToken[]; findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number; sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): Modes.IToken[]; @@ -1204,7 +1201,6 @@ export interface ILineTokens { getTokenCount(): number; getTokenStartIndex(tokenIndex:number): number; getTokenType(tokenIndex:number): string; - getTokenBracket(tokenIndex:number): Modes.Bracket; getTokenEndIndex(tokenIndex:number, textLength:number): number; /** diff --git a/src/vs/editor/common/model/modelLine.ts b/src/vs/editor/common/model/modelLine.ts index b118df9239aa85c8af27d0702443a2c3db8b497f..1892ab87957d967f126e58b0c264b5f16b0b9421 100644 --- a/src/vs/editor/common/model/modelLine.ts +++ b/src/vs/editor/common/model/modelLine.ts @@ -171,9 +171,8 @@ export class ModelLine { if (currentTokenStartIndex > 0 && delta !== 0) { // adjust token's `startIndex` by `delta` let deflatedType = (tokens[tokensIndex] / BIN.TYPE_OFFSET) & BIN.TYPE_MASK; - let deflatedBracket = (tokens[tokensIndex] / BIN.BRACKET_OFFSET) & BIN.BRACKET_MASK; let newStartIndex = Math.max(minimumAllowedIndex, currentTokenStartIndex + delta); - let newToken = deflatedBracket * BIN.BRACKET_OFFSET + deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET; + let newToken = deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET; if (delta < 0) { // pop all previous tokens that have become `collapsed` @@ -439,9 +438,8 @@ export class ModelLine { let deflatedStartIndex = (token / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK; let deflatedType = (token / BIN.TYPE_OFFSET) & BIN.TYPE_MASK; - let deflatedBracket = (token / BIN.BRACKET_OFFSET) & BIN.BRACKET_MASK; let newStartIndex = deflatedStartIndex + thisTextLength; - let newToken = deflatedBracket * BIN.BRACKET_OFFSET + deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET; + let newToken = deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET; otherTokens[i] = newToken; } @@ -618,7 +616,7 @@ function toLineTokens(map:EditorCommon.ITokensInflatorMap, tokens:Modes.IToken[] return null; } } else { - if (tokens[0].startIndex === 0 && tokens[0].type === '' && !tokens[0].bracket) { + if (tokens[0].startIndex === 0 && tokens[0].type === '') { return null; } } @@ -628,7 +626,6 @@ function toLineTokens(map:EditorCommon.ITokensInflatorMap, tokens:Modes.IToken[] var getStartIndex = EditorCommon.LineTokensBinaryEncoding.getStartIndex; var getType = EditorCommon.LineTokensBinaryEncoding.getType; -var getBracket = EditorCommon.LineTokensBinaryEncoding.getBracket; var findIndexOfOffset = EditorCommon.LineTokensBinaryEncoding.findIndexOfOffset; export class LineTokens implements EditorCommon.ILineTokens { @@ -669,10 +666,6 @@ export class LineTokens implements EditorCommon.ILineTokens { return getType(this.map, this._tokens[tokenIndex]); } - public getTokenBracket(tokenIndex:number): Modes.Bracket { - return getBracket(this._tokens[tokenIndex]); - } - public getTokenEndIndex(tokenIndex:number, textLength:number): number { if (tokenIndex + 1 < this._tokens.length) { return getStartIndex(this._tokens[tokenIndex + 1]); @@ -714,10 +707,6 @@ class EmptyLineTokens implements EditorCommon.ILineTokens { return Strings.empty; } - public getTokenBracket(tokenIndex:number): Modes.Bracket { - return 0; - } - public getTokenEndIndex(tokenIndex:number, textLength:number): number { return 0; } @@ -756,10 +745,6 @@ export class DefaultLineTokens implements EditorCommon.ILineTokens { return Strings.empty; } - public getTokenBracket(tokenIndex:number): Modes.Bracket { - return 0; - } - public getTokenEndIndex(tokenIndex:number, textLength:number): number { return textLength; } diff --git a/src/vs/editor/common/model/textModelWithTokens.ts b/src/vs/editor/common/model/textModelWithTokens.ts index 68bbe2e023f019508c94201602149422ab275d5a..d3b4a5e7e9cb8031b973366507d832966d2d386b 100644 --- a/src/vs/editor/common/model/textModelWithTokens.ts +++ b/src/vs/editor/common/model/textModelWithTokens.ts @@ -166,10 +166,6 @@ class LineContext implements Modes.ILineContext { return this._lineTokens.getTokenType(tokenIndex); } - public getTokenBracket(tokenIndex:number): Modes.Bracket { - return this._lineTokens.getTokenBracket(tokenIndex); - } - public getTokenText(tokenIndex:number): string { var startIndex = this._lineTokens.getTokenStartIndex(tokenIndex); var endIndex = this._lineTokens.getTokenEndIndex(tokenIndex, this._text.length); diff --git a/src/vs/editor/common/model/textModelWithTokensHelpers.ts b/src/vs/editor/common/model/textModelWithTokensHelpers.ts index 0cbb7aed3ad0b531e74c3c36725bbf31747789bd..a4991351bbdcb42ffdea89f865410b8f7a484432 100644 --- a/src/vs/editor/common/model/textModelWithTokensHelpers.ts +++ b/src/vs/editor/common/model/textModelWithTokensHelpers.ts @@ -10,6 +10,7 @@ import Modes = require('vs/editor/common/modes'); import EditorCommon = require('vs/editor/common/editorCommon'); import {Arrays} from 'vs/editor/common/core/arrays'; import Errors = require('vs/base/common/errors'); +import {getBracketFor} from 'vs/editor/common/modes/supports'; export interface ITextSource { @@ -29,7 +30,6 @@ export interface ITextSource { } var getType = EditorCommon.LineTokensBinaryEncoding.getType; -var getBracket = EditorCommon.LineTokensBinaryEncoding.getBracket; var getStartIndex = EditorCommon.LineTokensBinaryEncoding.getStartIndex; export interface INonWordTokenMap { @@ -241,24 +241,23 @@ export class BracketsHelper { private static _findMatchingBracketUp(textSource:ITextSource, type:string, lineNumber:number, tokenIndex:number, initialCount:number): Range { - var i:number, - end:number, - j:number, - count = initialCount; + let mode = textSource.getMode(); + let count = initialCount; - for (i = lineNumber; i >= 1; i--) { + for (let i = lineNumber; i >= 1; i--) { - var lineTokens = textSource.getLineTokens(i, false), - tokens = lineTokens.getBinaryEncodedTokens(), - tokensMap = lineTokens.getBinaryEncodedTokensMap(), - lineText = textSource.getLineContent(i); + let lineTokens = textSource.getLineTokens(i, false); + let tokens = lineTokens.getBinaryEncodedTokens(); + let tokensMap = lineTokens.getBinaryEncodedTokensMap(); + let lineText = textSource.getLineContent(i); - for (j = (i === lineNumber ? tokenIndex : tokens.length) - 1; j >= 0; j--) { + for (let j = (i === lineNumber ? tokenIndex : tokens.length) - 1; j >= 0; j--) { if (getType(tokensMap, tokens[j]) === type) { - count += BracketsHelper._sign(getBracket(tokens[j])); + let start = getStartIndex(tokens[j]); + let end = (j === tokens.length - 1 ? lineText.length : getStartIndex(tokens[j + 1])); + count += BracketsHelper._sign(getBracketFor(type, lineText.substring(start, end), mode)); if (count === 0) { - end = (j === tokens.length - 1 ? lineText.length : getStartIndex(tokens[j + 1])); - return new Range(i, getStartIndex(tokens[j]) + 1, i, end + 1); + return new Range(i, start + 1, i, end + 1); } } } @@ -268,14 +267,10 @@ export class BracketsHelper { private static _findMatchingBracketDown(textSource:ITextSource, type:string, lineNumber:number, tokenIndex:number, inaccurateResultAcceptable:boolean): { range:Range; isAccurate:boolean; } { - var i:number, - len:number, - end:number, - j:number, - lenJ:number, - count = 1; + let mode = textSource.getMode(); + let count = 1; - for (i = lineNumber, len = textSource.getLineCount(); i <= len; i++) { + for (let i = lineNumber, len = textSource.getLineCount(); i <= len; i++) { if (inaccurateResultAcceptable && !textSource._lineIsTokenized(i)) { return { range: null, @@ -288,13 +283,14 @@ export class BracketsHelper { tokensMap = lineTokens.getBinaryEncodedTokensMap(), lineText = textSource.getLineContent(i); - for (j = (i === lineNumber ? tokenIndex + 1 : 0), lenJ = tokens.length; j < lenJ; j++) { + for (let j = (i === lineNumber ? tokenIndex + 1 : 0), lenJ = tokens.length; j < lenJ; j++) { if (getType(tokensMap, tokens[j]) === type) { - count += BracketsHelper._sign(getBracket(tokens[j])); + let start = getStartIndex(tokens[j]); + let end = (j === tokens.length - 1 ? lineText.length : getStartIndex(tokens[j + 1])); + count += BracketsHelper._sign(getBracketFor(type, lineText.substring(start, end), mode)); if (count === 0) { - end = (j === tokens.length - 1 ? lineText.length : getStartIndex(tokens[j + 1])); return { - range: new Range(i, getStartIndex(tokens[j]) + 1, i, end + 1), + range: new Range(i, start + 1, i, end + 1), isAccurate: true }; } @@ -332,6 +328,9 @@ export class BracketsHelper { } public static matchBracket(textSource:ITextSource, position:EditorCommon.IPosition, inaccurateResultAcceptable:boolean): EditorCommon.IMatchBracketResult { + + let mode = textSource.getMode(); + if (inaccurateResultAcceptable && !textSource._lineIsTokenized(position.lineNumber)) { return { brackets: null, @@ -364,9 +363,8 @@ export class BracketsHelper { token = tokens[i]; tokenStartIndex = getStartIndex(token); tokenType = getType(tokensMap, token); - tokenBracket = getBracket(token); - end = i === len - 1 ? lineText.length : getStartIndex(tokens[i + 1]); + tokenBracket = getBracketFor(tokenType, lineText.substring(tokenStartIndex, end), mode); if (tokenStartIndex <= columnIndex && columnIndex <= end) { if (tokenBracket < 0) { diff --git a/src/vs/editor/common/model/tokensBinaryEncoding.ts b/src/vs/editor/common/model/tokensBinaryEncoding.ts index 3a6e06718d35d813ce063b732d708c72b4036637..9589754ba557e5c74d4e8299b4ce3f5309eab3bc 100644 --- a/src/vs/editor/common/model/tokensBinaryEncoding.ts +++ b/src/vs/editor/common/model/tokensBinaryEncoding.ts @@ -12,30 +12,25 @@ import Errors = require('vs/base/common/errors'); class InflatedToken implements Modes.IToken { startIndex:number; type:string; - bracket:Modes.Bracket; - constructor(startIndex:number, type:string, bracket:Modes.Bracket) { + constructor(startIndex:number, type:string) { this.startIndex = startIndex; this.type = type; - this.bracket = bracket; } public toString(): string { - return '{ ' + this.startIndex + ', \'' + this.type + '\', ' + this.bracket + '}'; + return '{ ' + this.startIndex + ', \'' + this.type + '\'}'; } } export var START_INDEX_MASK = 0xffffffff; export var TYPE_MASK = 0xffff; -export var BRACKET_MASK = 0xff; export var START_INDEX_OFFSET = 1; export var TYPE_OFFSET = Math.pow(2, 32); -export var BRACKET_OFFSET = Math.pow(2, 48); var DEFAULT_TOKEN = { startIndex: 0, - type: '', - bracket: 0 + type: '' }; var INFLATED_TOKENS_EMPTY_TEXT = []; var DEFLATED_TOKENS_EMPTY_TEXT = []; @@ -46,14 +41,13 @@ export function deflateArr(map:EditorCommon.ITokensInflatorMap, tokens:Modes.ITo if (tokens.length === 0) { return DEFLATED_TOKENS_EMPTY_TEXT; } - if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type && !tokens[0].bracket) { + if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type) { return DEFLATED_TOKENS_NON_EMPTY_TEXT; } var i:number, len:number, deflatedToken:number, - deflatedBracket:number, deflated:number, token:Modes.IToken, inflateMap = map._inflate, @@ -80,11 +74,6 @@ export function deflateArr(map:EditorCommon.ITokensInflatorMap, tokens:Modes.ITo inflateMap.push(token.type); } - deflatedBracket = token.bracket; - if (deflatedBracket < 0) { - deflatedBracket = 2; - } - // http://stackoverflow.com/a/2803010 // All numbers in JavaScript are actually IEEE-754 compliant floating-point doubles. // These have a 53-bit mantissa which should mean that any integer value with a magnitude @@ -96,10 +85,9 @@ export function deflateArr(map:EditorCommon.ITokensInflatorMap, tokens:Modes.ITo // 32 bits for startIndex => up to 2^32 = 4,294,967,296 // 16 bits for token => up to 2^16 = 65,536 - // 2 bits for bracket => up to 2^2 = 4 - // [bracket][token][startIndex] - deflated = deflatedBracket * BRACKET_OFFSET + deflatedToken * TYPE_OFFSET + token.startIndex * START_INDEX_OFFSET; + // [token][startIndex] + deflated = deflatedToken * TYPE_OFFSET + token.startIndex * START_INDEX_OFFSET; result[i] = deflated; @@ -116,13 +104,8 @@ export function inflate(map:EditorCommon.ITokensInflatorMap, binaryEncodedToken: var startIndex = (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK; var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK; - var deflatedBracket = (binaryEncodedToken / BRACKET_OFFSET) & BRACKET_MASK; - - if (deflatedBracket === 2) { - deflatedBracket = -1; - } - return new InflatedToken(startIndex, map._inflate[deflatedType], deflatedBracket); + return new InflatedToken(startIndex, map._inflate[deflatedType]); } export function getStartIndex(binaryEncodedToken:number): number { @@ -137,16 +120,6 @@ export function getType(map:EditorCommon.ITokensInflatorMap, binaryEncodedToken: return map._inflate[deflatedType]; } -export function getBracket(binaryEncodedToken:number): Modes.Bracket { - var deflatedBracket = (binaryEncodedToken / BRACKET_OFFSET) & BRACKET_MASK; - - if (deflatedBracket === 2) { - deflatedBracket = -1; - } - - return deflatedBracket; -} - export function inflateArr(map:EditorCommon.ITokensInflatorMap, binaryEncodedTokens:number[]): Modes.IToken[] { if (binaryEncodedTokens.length === 0) { return INFLATED_TOKENS_EMPTY_TEXT; @@ -160,7 +133,6 @@ export function inflateArr(map:EditorCommon.ITokensInflatorMap, binaryEncodedTok len:number, deflated:number, startIndex:number, - deflatedBracket:number, deflatedType:number, inflateMap = map._inflate; @@ -169,13 +141,8 @@ export function inflateArr(map:EditorCommon.ITokensInflatorMap, binaryEncodedTok startIndex = (deflated / START_INDEX_OFFSET) & START_INDEX_MASK; deflatedType = (deflated / TYPE_OFFSET) & TYPE_MASK; - deflatedBracket = (deflated / BRACKET_OFFSET) & BRACKET_MASK; - - if (deflatedBracket === 2) { - deflatedBracket = -1; - } - result[i] = new InflatedToken(startIndex, inflateMap[deflatedType], deflatedBracket); + result[i] = new InflatedToken(startIndex, inflateMap[deflatedType]); } return result; @@ -200,15 +167,13 @@ export function sliceAndInflate(map:EditorCommon.ITokensInflatorMap, binaryEncod originalStartIndex:number, newStartIndex:number, deflatedType:number, - deflatedBracket:number, result: Modes.IToken[] = [], inflateMap = map._inflate; originalToken = binaryEncodedTokens[startIndex]; deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK; - deflatedBracket = (originalToken / BRACKET_OFFSET) & BRACKET_MASK; newStartIndex = 0; - result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType], deflatedBracket)); + result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType])); for (i = startIndex + 1, len = binaryEncodedTokens.length; i < len; i++) { originalToken = binaryEncodedTokens[i]; @@ -219,9 +184,8 @@ export function sliceAndInflate(map:EditorCommon.ITokensInflatorMap, binaryEncod } deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK; - deflatedBracket = (originalToken / BRACKET_OFFSET) & BRACKET_MASK; newStartIndex = originalStartIndex - startOffset + deltaStartIndex; - result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType], deflatedBracket)); + result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType])); } return result; diff --git a/src/vs/editor/common/modes.ts b/src/vs/editor/common/modes.ts index 7a7125a8eeee144d1a51dd4aa35724ad611b4ce4..dd47bd864b0852c6b66b24ccaa6e919ff1096321 100644 --- a/src/vs/editor/common/modes.ts +++ b/src/vs/editor/common/modes.ts @@ -176,7 +176,6 @@ export interface ILineContext { getTokenCount(): number; getTokenStartIndex(tokenIndex:number): number; getTokenType(tokenIndex:number): string; - getTokenBracket(tokenIndex:number): Bracket; getTokenText(tokenIndex:number): string; getTokenEndIndex(tokenIndex:number): number; findIndexOfOffset(offset:number): number; @@ -332,7 +331,7 @@ export interface IMode { export interface IToken { startIndex:number; type:string; - bracket:Bracket; + bracket?:Bracket; } export interface IModeTransition { diff --git a/src/vs/editor/common/modes/nullMode.ts b/src/vs/editor/common/modes/nullMode.ts index 12e199ee5b327650697bab7d46cfac015e292565..20644133a02137f61d4c4a6276244527247da16a 100644 --- a/src/vs/editor/common/modes/nullMode.ts +++ b/src/vs/editor/common/modes/nullMode.ts @@ -103,8 +103,7 @@ export function nullTokenize(mode: Modes.IMode, buffer:string, state: Modes.ISta var tokens:Modes.IToken[] = [ { startIndex: deltaOffset, - type: '', - bracket: Modes.Bracket.None + type: '' } ]; diff --git a/src/vs/editor/common/modes/supports.ts b/src/vs/editor/common/modes/supports.ts index 03eb23685982683b6cc5119e83fc0d9dfada4f16..71bb9bd5f35989cd18f34411c00c18968c999003 100644 --- a/src/vs/editor/common/modes/supports.ts +++ b/src/vs/editor/common/modes/supports.ts @@ -21,16 +21,14 @@ import {IDisposable} from 'vs/base/common/lifecycle'; export class Token implements Modes.IToken { public startIndex:number; public type:string; - public bracket:Modes.Bracket; - constructor(startIndex:number, type:string, bracket:Modes.Bracket) { + constructor(startIndex:number, type:string) { this.startIndex = startIndex; this.type = type; - this.bracket = bracket; } public toString(): string { - return '(' + this.startIndex + ', ' + this.type + ', ' + this.bracket + ')'; + return '(' + this.startIndex + ', ' + this.type + ')'; } } @@ -146,10 +144,6 @@ export class FilteredLineContext implements Modes.ILineContext { return this._actual.getTokenType(tokenIndex + this._firstTokenInModeIndex); } - public getTokenBracket(tokenIndex:number): Modes.Bracket { - return this._actual.getTokenBracket(tokenIndex + this._firstTokenInModeIndex); - } - public getTokenText(tokenIndex:number): string { return this._actual.getTokenText(tokenIndex + this._firstTokenInModeIndex); } @@ -357,7 +351,6 @@ export class TokenizationSupport extends AbstractSupport implements Modes.IToken } var maxPos = Math.min(stopAtOffset - deltaOffset, buffer.length); - var noneBracket = Modes.Bracket.None; while (lineStream.pos() < maxPos) { beforeTokenizeStreamPos = lineStream.pos(); @@ -379,7 +372,7 @@ export class TokenizationSupport extends AbstractSupport implements Modes.IToken } while (!tokenResult.type && tokenResult.type !== ''); if (previousType !== tokenResult.type || tokenResult.bracket || previousType === null) { - prependTokens.push(new Token(beforeTokenizeStreamPos + deltaOffset, tokenResult.type, tokenResult.bracket || noneBracket)); + prependTokens.push(new Token(beforeTokenizeStreamPos + deltaOffset, tokenResult.type)); } previousType = tokenResult.type; @@ -561,7 +554,16 @@ export class BracketElectricCharacterSupport extends AbstractSupport implements } } - +// TODO@Alex -> refactor to use `brackets` from language configuration +export function getBracketFor(tokenType:string, tokenText:string, mode:Modes.IMode): Modes.Bracket { + if (tokenText === '{' || tokenText === '(' || tokenText === '[') { + return Modes.Bracket.Open; + } + if (tokenText === '}' || tokenText === ')' || tokenText === ']') { + return Modes.Bracket.Close; + } + return Modes.Bracket.None; +} export interface IDeclarationContribution { tokens?: string[]; diff --git a/src/vs/editor/contrib/smartSelect/common/tokenTree.ts b/src/vs/editor/contrib/smartSelect/common/tokenTree.ts index 67e4c10b522739a9dc0b7692d4a2fd613f19ee1b..a1fafe1ddf82a743f1e2dc7450e3a55705432ca2 100644 --- a/src/vs/editor/contrib/smartSelect/common/tokenTree.ts +++ b/src/vs/editor/contrib/smartSelect/common/tokenTree.ts @@ -8,6 +8,7 @@ import EditorCommon = require('vs/editor/common/editorCommon'); import Modes = require('vs/editor/common/modes'); import {Range} from 'vs/editor/common/core/range'; import {Position} from 'vs/editor/common/core/position'; +import {getBracketFor} from 'vs/editor/common/modes/supports'; export class Node { @@ -108,6 +109,7 @@ class TokenScanner { private _currentLineNumber: number; private _currentTokenIndex: number; private _currentLineTokens: EditorCommon.ILineTokens; + private _currentLineText: string; constructor(model: EditorCommon.IModel) { this._model = model; @@ -127,22 +129,28 @@ class TokenScanner { if (!this._currentLineTokens) { // no tokens for this line this._currentLineTokens = this._model.getLineTokens(this._currentLineNumber); + this._currentLineText = this._model.getLineContent(this._currentLineNumber); this._currentTokenIndex = 0; } if (this._currentTokenIndex >= this._currentLineTokens.getTokenCount()) { // last token of line visited this._currentLineNumber += 1; this._currentLineTokens = null; + this._currentLineText = null; return this.next(); } + let tokenType = this._currentLineTokens.getTokenType(this._currentTokenIndex); + let tokenStartIndex = this._currentLineTokens.getTokenStartIndex(this._currentTokenIndex); + let tokenEndIndex = this._currentLineTokens.getTokenEndIndex(this._currentTokenIndex, this._currentLineText.length + 1); + let tokenText = this._currentLineText.substring(tokenStartIndex, tokenEndIndex); var token: Token = { - type: this._currentLineTokens.getTokenType(this._currentTokenIndex), - bracket: this._currentLineTokens.getTokenBracket(this._currentTokenIndex), + type: tokenType, + bracket: getBracketFor(tokenType, tokenText, this._model.getMode()), range: { startLineNumber: this._currentLineNumber, - startColumn: 1 + this._currentLineTokens.getTokenStartIndex(this._currentTokenIndex), + startColumn: 1 + tokenStartIndex, endLineNumber: this._currentLineNumber, - endColumn: 1 + this._currentLineTokens.getTokenEndIndex(this._currentTokenIndex, this._model.getLineMaxColumn(this._currentLineNumber)) + endColumn: 1 + tokenEndIndex } }; // token.__debugContent = this._model.getValueInRange(token.range); diff --git a/src/vs/editor/node/textMate/TMSyntax.ts b/src/vs/editor/node/textMate/TMSyntax.ts index e92097a85df2900bdcb02a2715da73d56a3de098..9aaa7598ffde8dad9d633f9942679c1b03e5cbf6 100644 --- a/src/vs/editor/node/textMate/TMSyntax.ts +++ b/src/vs/editor/node/textMate/TMSyntax.ts @@ -162,10 +162,6 @@ class Tokenizer { modeTransitions: [{ startIndex: offsetDelta, mode: freshState.getMode() }], }; - let noBracket = Modes.Bracket.None, - openBracket = Modes.Bracket.Open, - closeBracket = Modes.Bracket.Close; - for (let tokenIndex = 0, len = textMateResult.tokens.length; tokenIndex < len; tokenIndex++) { let token = textMateResult.tokens[tokenIndex]; let tokenStartIndex = token.startIndex; @@ -175,7 +171,7 @@ class Tokenizer { if (t.isOpaqueToken) { // Should not do any smartness to detect brackets on this token - ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType, noBracket)); + ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType)); continue; } @@ -187,51 +183,44 @@ class Tokenizer { for (i = tokenStartIndex; i < tokenEndIndex; i++) { charCode = line.charCodeAt(i); isBracket = null; - bracketType = noBracket; switch (charCode) { case _openParen: // ( isBracket = 'delimiter.paren'; - bracketType = openBracket; break; case _closeParen: // ) isBracket = 'delimiter.paren'; - bracketType = closeBracket; break; case _openCurly: // { isBracket = 'delimiter.curly'; - bracketType = openBracket; break; case _closeCurly: // } isBracket = 'delimiter.curly'; - bracketType = closeBracket; break; case _openSquare: // [ isBracket = 'delimiter.square'; - bracketType = openBracket; break; case _closeSquare: // ] isBracket = 'delimiter.square'; - bracketType = closeBracket; break; } if (isBracket) { if (tokenStartIndex < i) { // push a token before character `i` - ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType, noBracket)); + ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType)); tokenStartIndex = i; } // push character `i` as a token - ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, isBracket + '.' + t.modeToken, bracketType)); + ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, isBracket + '.' + t.modeToken)); tokenStartIndex = i + 1; } } if (tokenStartIndex < tokenEndIndex) { // push the remaining text as a token - ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType, noBracket)); + ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType)); } } diff --git a/src/vs/editor/test/common/model/model.line.test.ts b/src/vs/editor/test/common/model/model.line.test.ts index df3d189def3f0b479532f7e836474624ec01b60d..bc172822d91c147efcd175995ffd4edfc452e8f3 100644 --- a/src/vs/editor/test/common/model/model.line.test.ts +++ b/src/vs/editor/test/common/model/model.line.test.ts @@ -248,7 +248,7 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { test('insertion on empty line', () => { var line = new ModelLine.ModelLine(1, 'some text'); var map = new TextModelWithTokens.TokensInflatorMap(); - line.setTokens(map, [{startIndex: 0, type:'bar', bracket:0}], null, []); + line.setTokens(map, [{startIndex: 0, type:'bar'}], null, []); line.applyEdits({}, [{startColumn:1, endColumn:10, text:'', forceMoveMarkers: false}]); line.setTokens(map, [], null, []); @@ -256,8 +256,7 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { line.applyEdits({}, [{startColumn:1, endColumn:1, text:'a', forceMoveMarkers: false}]); assertLineTokens(line.getTokens(), [{ startIndex: 0, - type:'', - bracket:0 + type:'' }]); }) @@ -265,9 +264,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 1, @@ -277,9 +276,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'aabcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 5, type: '2', bracket: 1 }, - { startIndex: 6, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 5, type: '2' }, + { startIndex: 6, type: '3' } ] ); }); @@ -288,9 +287,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'aabcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 5, type: '2', bracket: 1 }, - { startIndex: 6, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 5, type: '2' }, + { startIndex: 6, type: '3' } ], [{ startColumn: 2, @@ -300,9 +299,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'axabcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 6, type: '2', bracket: 1 }, - { startIndex: 7, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 6, type: '2' }, + { startIndex: 7, type: '3' } ] ); }); @@ -311,9 +310,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axabcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 6, type: '2', bracket: 1 }, - { startIndex: 7, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 6, type: '2' }, + { startIndex: 7, type: '3' } ], [{ startColumn: 3, @@ -323,9 +322,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'axstuabcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 9, type: '2', bracket: 1 }, - { startIndex: 10, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 9, type: '2' }, + { startIndex: 10, type: '3' } ] ); }); @@ -334,9 +333,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axstuabcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 9, type: '2', bracket: 1 }, - { startIndex: 10, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 9, type: '2' }, + { startIndex: 10, type: '3' } ], [{ startColumn: 10, @@ -346,9 +345,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'axstuabcd\t efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 10, type: '2', bracket: 1 }, - { startIndex: 11, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 10, type: '2' }, + { startIndex: 11, type: '3' } ] ); }); @@ -357,9 +356,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axstuabcd\t efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 10, type: '2', bracket: 1 }, - { startIndex: 11, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 10, type: '2' }, + { startIndex: 11, type: '3' } ], [{ startColumn: 12, @@ -369,9 +368,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'axstuabcd\t ddefgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 10, type: '2', bracket: 1 }, - { startIndex: 13, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 10, type: '2' }, + { startIndex: 13, type: '3' } ] ); }); @@ -380,9 +379,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axstuabcd\t ddefgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 10, type: '2', bracket: 1 }, - { startIndex: 13, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 10, type: '2' }, + { startIndex: 13, type: '3' } ], [{ startColumn: 18, @@ -392,9 +391,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'axstuabcd\t ddefghxyz', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 10, type: '2', bracket: 1 }, - { startIndex: 13, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 10, type: '2' }, + { startIndex: 13, type: '3' } ] ); }); @@ -403,9 +402,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axstuabcd\t ddefghxyz', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 10, type: '2', bracket: 1 }, - { startIndex: 13, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 10, type: '2' }, + { startIndex: 13, type: '3' } ], [{ startColumn: 1, @@ -415,9 +414,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'xaxstuabcd\t ddefghxyz', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 11, type: '2', bracket: 1 }, - { startIndex: 14, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 11, type: '2' }, + { startIndex: 14, type: '3' } ] ); }); @@ -426,9 +425,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'xaxstuabcd\t ddefghxyz', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 11, type: '2', bracket: 1 }, - { startIndex: 14, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 11, type: '2' }, + { startIndex: 14, type: '3' } ], [{ startColumn: 22, @@ -438,9 +437,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'xaxstuabcd\t ddefghxyzx', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 11, type: '2', bracket: 1 }, - { startIndex: 14, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 11, type: '2' }, + { startIndex: 14, type: '3' } ] ); }); @@ -449,9 +448,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'xaxstuabcd\t ddefghxyzx', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 11, type: '2', bracket: 1 }, - { startIndex: 14, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 11, type: '2' }, + { startIndex: 14, type: '3' } ], [{ startColumn: 2, @@ -461,9 +460,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'xaxstuabcd\t ddefghxyzx', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 11, type: '2', bracket: 1 }, - { startIndex: 14, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 11, type: '2' }, + { startIndex: 14, type: '3' } ] ); }); @@ -480,7 +479,7 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'a', [ - { startIndex: 0, type: '', bracket: 0 } + { startIndex: 0, type: '' } ] ); }); @@ -489,9 +488,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcdefghij', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 3, type: '2', bracket: 1 }, - { startIndex: 6, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 3, type: '2' }, + { startIndex: 6, type: '3' } ], [{ startColumn: 4, @@ -501,8 +500,8 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'abcghij', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 3, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 3, type: '3' } ] ); }); @@ -511,9 +510,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcdefghij', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 3, type: '2', bracket: 1 }, - { startIndex: 6, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 3, type: '2' }, + { startIndex: 6, type: '3' } ], [{ startColumn: 4, @@ -523,9 +522,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'abchellodefghij', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 8, type: '2', bracket: 1 }, - { startIndex: 11, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 8, type: '2' }, + { startIndex: 11, type: '3' } ] ); }); @@ -534,9 +533,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 1, @@ -546,9 +545,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'bcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 3, type: '2', bracket: 1 }, - { startIndex: 4, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 3, type: '2' }, + { startIndex: 4, type: '3' } ] ); }); @@ -557,9 +556,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 2, @@ -569,9 +568,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'ad efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 2, type: '2', bracket: 1 }, - { startIndex: 3, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 2, type: '2' }, + { startIndex: 3, type: '3' } ] ); }); @@ -580,9 +579,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 1, @@ -592,8 +591,8 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], ' efgh', [ - { startIndex: 0, type: '2', bracket: 1 }, - { startIndex: 1, type: '3', bracket: -1 } + { startIndex: 0, type: '2' }, + { startIndex: 1, type: '3' } ] ); }); @@ -602,9 +601,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 5, @@ -614,8 +613,8 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'abcdefgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '3' } ] ); }); @@ -624,9 +623,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 5, @@ -636,8 +635,8 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'abcdfgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '3' } ] ); }); @@ -646,9 +645,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 5, @@ -658,7 +657,7 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'abcd', [ - { startIndex: 0, type: '1', bracket: 0 } + { startIndex: 0, type: '1' } ] ); }); @@ -667,9 +666,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 1, @@ -686,9 +685,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 1, @@ -698,9 +697,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ] ); }); @@ -709,9 +708,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 1, @@ -721,9 +720,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'cd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 2, type: '2', bracket: 1 }, - { startIndex: 3, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 2, type: '2' }, + { startIndex: 3, type: '3' } ] ); }); @@ -732,9 +731,9 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], [{ startColumn: 5, @@ -744,7 +743,7 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'abcd', [ - { startIndex: 0, type: '1', bracket: 0 } + { startIndex: 0, type: '1' } ] ); }); @@ -753,11 +752,11 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'Hello world, ciao', [ - { startIndex: 0, type: 'hello', bracket: 0 }, - { startIndex: 5, type: '', bracket: 0 }, - { startIndex: 6, type: 'world', bracket: 0 }, - { startIndex: 11, type: '', bracket: 0 }, - { startIndex: 13, type: '', bracket: 0 }, + { startIndex: 0, type: 'hello' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'world' }, + { startIndex: 11, type: '' }, + { startIndex: 13, type: '' }, ], [{ startColumn: 1, @@ -767,11 +766,11 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'Hi world, ciao', [ - { startIndex: 0, type: 'hello', bracket: 0 }, - { startIndex: 2, type: '', bracket: 0 }, - { startIndex: 3, type: 'world', bracket: 0 }, - { startIndex: 8, type: '', bracket: 0 }, - { startIndex: 10, type: '', bracket: 0 }, + { startIndex: 0, type: 'hello' }, + { startIndex: 2, type: '' }, + { startIndex: 3, type: 'world' }, + { startIndex: 8, type: '' }, + { startIndex: 10, type: '' }, ] ); }); @@ -780,11 +779,11 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { testLineEditTokens( 'Hello world, ciao', [ - { startIndex: 0, type: 'hello', bracket: 0 }, - { startIndex: 5, type: '', bracket: 0 }, - { startIndex: 6, type: 'world', bracket: 0 }, - { startIndex: 11, type: '', bracket: 0 }, - { startIndex: 13, type: '', bracket: 0 }, + { startIndex: 0, type: 'hello' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'world' }, + { startIndex: 11, type: '' }, + { startIndex: 13, type: '' }, ], [{ startColumn: 1, @@ -799,11 +798,11 @@ suite('Editor Model - ModelLine.applyEdits text & tokens', () => { }], 'Hi wmy friends, ciao', [ - { startIndex: 0, type: 'hello', bracket: 0 }, - { startIndex: 2, type: '', bracket: 0 }, - { startIndex: 3, type: 'world', bracket: 0 }, - { startIndex: 14, type: '', bracket: 0 }, - { startIndex: 16, type: '', bracket: 0 }, + { startIndex: 0, type: 'hello' }, + { startIndex: 2, type: '' }, + { startIndex: 3, type: 'world' }, + { startIndex: 14, type: '' }, + { startIndex: 16, type: '' }, ] ); }); @@ -825,9 +824,9 @@ suite('Editor Model - ModelLine.split text & tokens', () => { testLineSplitTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], 1, '', @@ -840,17 +839,17 @@ suite('Editor Model - ModelLine.split text & tokens', () => { testLineSplitTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], 10, 'abcd efgh', '', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ] ); }); @@ -859,15 +858,15 @@ suite('Editor Model - ModelLine.split text & tokens', () => { testLineSplitTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], 5, 'abcd', ' efgh', [ - { startIndex: 0, type: '1', bracket: 0 } + { startIndex: 0, type: '1' } ] ); }); @@ -876,16 +875,16 @@ suite('Editor Model - ModelLine.split text & tokens', () => { testLineSplitTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], 6, 'abcd ', 'efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' } ] ); }); @@ -911,17 +910,17 @@ suite('Editor Model - ModelLine.append text & tokens', () => { testLineAppendTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], '', [], 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ] ); }); @@ -932,15 +931,15 @@ suite('Editor Model - ModelLine.append text & tokens', () => { [], 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ] ); }); @@ -949,24 +948,24 @@ suite('Editor Model - ModelLine.append text & tokens', () => { testLineAppendTokens( 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ], 'abcd efgh', [ - { startIndex: 0, type: '4', bracket: 0 }, - { startIndex: 4, type: '5', bracket: 1 }, - { startIndex: 5, type: '6', bracket: -1 } + { startIndex: 0, type: '4' }, + { startIndex: 4, type: '5' }, + { startIndex: 5, type: '6' } ], 'abcd efghabcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 }, - { startIndex: 9, type: '4', bracket: 0 }, - { startIndex: 13, type: '5', bracket: 1 }, - { startIndex: 14, type: '6', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' }, + { startIndex: 9, type: '4' }, + { startIndex: 13, type: '5' }, + { startIndex: 14, type: '6' } ] ); }); @@ -975,18 +974,18 @@ suite('Editor Model - ModelLine.append text & tokens', () => { testLineAppendTokens( 'abcd ', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' } ], 'efgh', [ - { startIndex: 0, type: '3', bracket: -1 } + { startIndex: 0, type: '3' } ], 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ] ); }); @@ -995,18 +994,18 @@ suite('Editor Model - ModelLine.append text & tokens', () => { testLineAppendTokens( 'abcd', [ - { startIndex: 0, type: '1', bracket: 0 }, + { startIndex: 0, type: '1' }, ], ' efgh', [ - { startIndex: 0, type: '2', bracket: 1 }, - { startIndex: 1, type: '3', bracket: -1 } + { startIndex: 0, type: '2' }, + { startIndex: 1, type: '3' } ], 'abcd efgh', [ - { startIndex: 0, type: '1', bracket: 0 }, - { startIndex: 4, type: '2', bracket: 1 }, - { startIndex: 5, type: '3', bracket: -1 } + { startIndex: 0, type: '1' }, + { startIndex: 4, type: '2' }, + { startIndex: 5, type: '3' } ] ); }); diff --git a/src/vs/editor/test/common/modes/tokenization.test.ts b/src/vs/editor/test/common/modes/tokenization.test.ts index 3854a550cef54c682d06b995f3ae3c1508411528..d52d7b99ea9dc91f681f09a0d3db96580fd926eb 100644 --- a/src/vs/editor/test/common/modes/tokenization.test.ts +++ b/src/vs/editor/test/common/modes/tokenization.test.ts @@ -177,7 +177,6 @@ function assertTokens(actual:modes.IToken[], expected:ITestToken[], message?:str for (var i = 0; i < expected.length; i++) { assert.equal(actual[i].startIndex, expected[i].startIndex, 'startIndex mismatch'); assert.equal(actual[i].type, expected[i].type, 'type mismatch'); - assert.equal(actual[i].bracket, expected[i].bracket ? expected[i].bracket : modes.Bracket.None, 'bracket mismatch'); } }; diff --git a/src/vs/editor/test/common/modesTestUtils.ts b/src/vs/editor/test/common/modesTestUtils.ts index cdb3a85aa35859799e88206e9611d2bef3a5acb2..06ee9ba05493a5af2b7c9a63059ce9e564975a64 100644 --- a/src/vs/editor/test/common/modesTestUtils.ts +++ b/src/vs/editor/test/common/modesTestUtils.ts @@ -48,7 +48,7 @@ export function createLineContextFromTokenText(tokens: TokenText[]): modes.ILine var indexSoFar = 0; for (var i = 0; i < tokens.length; ++i){ - processedTokens.push({ startIndex: indexSoFar, type: tokens[i].type, bracket: (tokens[i].bracket ? tokens[i].bracket : modes.Bracket.None) }); + processedTokens.push({ startIndex: indexSoFar, type: tokens[i].type }); line += tokens[i].text; indexSoFar += tokens[i].text.length; } @@ -95,10 +95,6 @@ class TestLineContext implements modes.ILineContext { return this._tokens[tokenIndex].type; } - public getTokenBracket(tokenIndex:number): modes.Bracket { - return this._tokens[tokenIndex].bracket; - } - public findIndexOfOffset(offset:number): number { return Arrays.findIndexInSegmentsArray(this._tokens, offset); } diff --git a/src/vs/editor/test/common/modesUtil.ts b/src/vs/editor/test/common/modesUtil.ts index 1d67d1e4f18cd8002053c434c831f394d5dc474b..27e29c0d1798395c63bf2d22e70b8c9f0ad50598 100644 --- a/src/vs/editor/test/common/modesUtil.ts +++ b/src/vs/editor/test/common/modesUtil.ts @@ -72,7 +72,7 @@ export function assertTokenization(tokenizationSupport: modes.ITokenizationSuppo assert.ok(true, tests[i].line); var result = tokenizationSupport.tokenize(tests[i].line, state); if (tests[i].tokens) { - assert.deepEqual(generateRelaxedTokens(result.tokens, tests[i].tokens), tests[i].tokens, JSON.stringify(result.tokens, null, '\t')); + assert.deepEqual(toRelaxedTokens(result.tokens), toRelaxedTokens(tests[i].tokens), JSON.stringify(result.tokens, null, '\t')); } state = result.endState; @@ -151,24 +151,13 @@ export function executeMonarchTokenizationTests(name:string, language:monarchTyp executeTests(tokenizationSupport, tests); } -function generateRelaxedTokens(actualTokens: modes.IToken[], expectedTokens: IRelaxedToken[]): IRelaxedToken[] { - var r = actualTokens.map((token, index) => { - // Remove bracket if it's missing in expectedTokens too - if (expectedTokens[index] && typeof expectedTokens[index].bracket !== 'undefined') { - return { - startIndex: token.startIndex, - type: token.type, - bracket: token.bracket - }; - } else { - return { - startIndex: token.startIndex, - type: token.type - }; - } +function toRelaxedTokens(tokens: modes.IToken[]): IRelaxedToken[] { + return tokens.map((t) => { + return { + startIndex: t.startIndex, + type: t.type + }; }); - - return r; } function executeTest(tokenizationSupport: modes.ITokenizationSupport, tests:ITestItem[]): void { @@ -187,5 +176,5 @@ function executeTest(tokenizationSupport: modes.ITokenizationSupport, tests:ITes } function assertTokens(actual:modes.IToken[], expected:IRelaxedToken[], message?:string): void { - assert.deepEqual(generateRelaxedTokens(actual, expected), expected, message + ': ' + JSON.stringify(actual, null, '\t')); + assert.deepEqual(toRelaxedTokens(actual), toRelaxedTokens(expected), message + ': ' + JSON.stringify(actual, null, '\t')); } \ No newline at end of file diff --git a/src/vs/languages/json/common/features/tokenization.ts b/src/vs/languages/json/common/features/tokenization.ts index dd2f041867e3962c3a383cc12b911fce41d277ec..06000871548e06a5f2330f757be0b646b0e832b9 100644 --- a/src/vs/languages/json/common/features/tokenization.ts +++ b/src/vs/languages/json/common/features/tokenization.ts @@ -181,8 +181,7 @@ function tokenize(mode:Modes.IMode, comments:boolean, line:string, state:JSONSta ret.endState = new JSONState(state.getMode(), state.getStateData(), scanner.getTokenError(), lastWasColon); ret.tokens.push({ startIndex: offset, - type: type, - bracket: bracket + type: type }); } diff --git a/src/vs/languages/typescript/common/features/tokenization.ts b/src/vs/languages/typescript/common/features/tokenization.ts index c939eb5857eedd6c8db07900649eaedbfd853e0b..ee158e52d8b2f0e6d3d63951215181fcd67f15ca 100644 --- a/src/vs/languages/typescript/common/features/tokenization.ts +++ b/src/vs/languages/typescript/common/features/tokenization.ts @@ -99,7 +99,7 @@ function tokenize(bracketTypeTable: { [i: number]: string }, tokenTypeTable: { [ function appendFn(startIndex:number, type:string, bracket:Modes.Bracket):void { if(ret.tokens.length === 0 || bracket !== void 0 || arrays.tail(ret.tokens).type !== type) { - ret.tokens.push(new supports.Token(startIndex, type, bracket || Modes.Bracket.None)); + ret.tokens.push(new supports.Token(startIndex, type)); } } diff --git a/src/vs/languages/typescript/test/common/tokenization.test.ts b/src/vs/languages/typescript/test/common/tokenization.test.ts index ab3aff8f8ced31c0625865a1916aa75699b41628..f691e49fb120d80a0fcb6a113542d647b339d048 100644 --- a/src/vs/languages/typescript/test/common/tokenization.test.ts +++ b/src/vs/languages/typescript/test/common/tokenization.test.ts @@ -4,8 +4,8 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; +import 'vs/languages/javascript/common/javascript.contribution'; -import javascriptMode = require('vs/languages/javascript/common/javascript'); import EditorCommon = require('vs/editor/common/editorCommon'); import Modes = require('vs/editor/common/modes'); import modesUtil = require('vs/editor/test/common/modesUtil');