Skip to content

V
vscode

xxadev / vscode
与 Fork 源项目一致

从无法访问的项目Fork

通知 2
Star 0
Fork 0
V
vscode

体验新版 GitCode，发现更多精彩内容 >>

提交 36972d22 编写于 作者: A Alex Dima
浏览文件
操作

Implement directly ITokenizationSupport in MonarchLexer

上级 92dfc35a
展开全部 隐藏空白更改
内联 并排
Showing with 568 addition and 1390 deletion
src/vs/editor/common/modes.ts
浏览文件 @ 36972d22
......@@ -24,8 +24,6 @@ export interface IState {
clone(): IState;
equals(other: IState): boolean;
getModeId(): string;
getStateData(): IState;
setStateData(state: IState): void;
}
/**
......
src/vs/editor/common/modes/abstractState.ts 已删除 100644 → 0
浏览文件 @ 92dfc35a
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { IState } from 'vs/editor/common/modes';
import { LineStream } from 'vs/editor/common/modes/lineStream';
/**
* @internal
*/
export interface ITokenizationResult {
type?: string;
dontMergeWithPrev?: boolean;
nextState?: AbstractState;
}
export abstract class AbstractState implements IState {
_abstractStateBrand: void;
private modeId: string;
private stateData: IState;
constructor(modeId: string, stateData: IState = null) {
this.modeId = modeId;
this.stateData = stateData;
}
public getModeId(): string {
return this.modeId;
}
public clone(): AbstractState {
var result: AbstractState = this.makeClone();
result.initializeFrom(this);
return result;
}
protected abstract makeClone(): AbstractState;
protected initializeFrom(other: AbstractState): void {
this.stateData = other.stateData !== null ? other.stateData.clone() : null;
}
public getStateData(): IState {
return this.stateData;
}
public setStateData(state: IState): void {
this.stateData = state;
}
public equals(other: IState): boolean {
if (other === null || this.modeId !== other.getModeId()) {
return false;
}
if (other instanceof AbstractState) {
return AbstractState.safeEquals(this.stateData, other.stateData);
}
return false;
}
public abstract tokenize(stream: LineStream): ITokenizationResult;
public static safeEquals(a: IState, b: IState): boolean {
if (a === null && b === null) {
return true;
}
if (a === null || b === null) {
return false;
}
return a.equals(b);
}
public static safeClone(state: IState): IState {
if (state) {
return state.clone();
}
return null;
}
}
src/vs/editor/common/modes/lineStream.ts 已删除 100644 → 0
浏览文件 @ 92dfc35a
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
/**
* A LineStream is a character & token stream abstraction over a line of text. It
* is never multi-line. The stream can be navigated character by character, or
* token by token, given some token rules.
* @internal
*/
export class LineStream {
private _source: string;
private _sourceLength: number;
private _pos: number;
constructor(source: string) {
this._source = source;
this._sourceLength = source.length;
this._pos = 0;
}
/**
* Returns the current character position of the stream on the line.
*/
public pos(): number {
return this._pos;
}
/**
* Returns true iff the stream is at the end of the line.
*/
public eos() {
return this._pos >= this._sourceLength;
}
/**
* Returns the next character in the stream.
*/
public peek(): string {
// Check EOS
if (this._pos >= this._sourceLength) {
throw new Error('Stream is at the end');
}
return this._source[this._pos];
}
/**
* Advances the stream by `n` characters.
*/
public advance(n: number): void {
if (n === 0) {
return;
}
this._pos += n;
}
/**
* Advances the stream until the end of the line.
*/
public advanceToEOS(): string {
const oldPos = this._pos;
this._pos = this._sourceLength;
return this._source.substring(oldPos, this._pos);
}
/**
* Brings the stream back `n` characters.
*/
public goBack(n: number) {
this._pos -= n;
}
}
src/vs/editor/common/modes/nullMode.ts
浏览文件 @ 36972d22
......@@ -7,53 +7,28 @@
import { IState, ILineTokens } from 'vs/editor/common/modes';
import { ModeTransition } from 'vs/editor/common/core/modeTransition';
import { Token } from 'vs/editor/common/core/token';
import { ITokenizationResult } from 'vs/editor/common/modes/abstractState';
import { LineStream } from 'vs/editor/common/modes/lineStream';
export class NullState implements IState {
private modeId: string;
private stateData: IState;
private readonly _modeId: string;
constructor(modeId: string, stateData: IState) {
this.modeId = modeId;
this.stateData = stateData;
constructor(modeId: string) {
this._modeId = modeId;
}
public clone(): IState {
let stateDataClone: IState = (this.stateData ? this.stateData.clone() : null);
return new NullState(this.modeId, stateDataClone);
return this;
}
public equals(other: IState): boolean {
if (this.modeId !== other.getModeId()) {
return false;
}
let otherStateData = other.getStateData();
if (!this.stateData && !otherStateData) {
return true;
}
if (this.stateData && otherStateData) {
return this.stateData.equals(otherStateData);
}
return false;
return (
other instanceof NullState
&& this._modeId === other._modeId
);
}
public getModeId(): string {
return this.modeId;
}
public tokenize(stream: LineStream): ITokenizationResult {
stream.advanceToEOS();
return { type: '' };
}
public getStateData(): IState {
return this.stateData;
}
public setStateData(stateData: IState): void {
this.stateData = stateData;
return this._modeId;
}
}
......
src/vs/editor/common/modes/supports/tokenizationSupport.ts 已删除 100644 → 0
浏览文件 @ 92dfc35a
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { IDisposable } from 'vs/base/common/lifecycle';
import * as modes from 'vs/editor/common/modes';
import { LineStream } from 'vs/editor/common/modes/lineStream';
import { NullState, nullTokenize, NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
import { Token } from 'vs/editor/common/core/token';
import { ModeTransition } from 'vs/editor/common/core/modeTransition';
import { IModeService } from 'vs/editor/common/services/modeService';
import { AbstractState, ITokenizationResult } from 'vs/editor/common/modes/abstractState';
export interface ILeavingNestedModeData {
/**
* The part of the line that will be tokenized by the nested mode
*/
nestedModeBuffer: string;
/**
* The part of the line that will be tokenized by the parent mode when it continues after the nested mode
*/
bufferAfterNestedMode: string;
/**
* The state that will be used for continuing tokenization by the parent mode after the nested mode
*/
stateAfterNestedMode: AbstractState;
}
export interface IModeLocator {
getMode(mimetypeOrModeId: string): modes.IMode;
}
export interface ITokenizationCustomization {
getInitialState(): AbstractState;
enterNestedMode?: (state: AbstractState) => boolean;
getNestedMode?: (state: AbstractState, locator: IModeLocator) => modes.IMode;
/**
* Return null if the line does not leave the nested mode
*/
getLeavingNestedModeData?: (line: string, state: modes.IState) => ILeavingNestedModeData;
}
function isFunction(something) {
return typeof something === 'function';
}
export class TokenizationSupport implements modes.ITokenizationSupport, IDisposable {
static MAX_EMBEDDED_LEVELS = 5;
private customization: ITokenizationCustomization;
private defaults: {
enterNestedMode: boolean;
getNestedMode: boolean;
getLeavingNestedModeData: boolean;
};
private supportsNestedModes: boolean;
private _modeService: IModeService;
private _modeId: string;
private _embeddedModes: { [modeId: string]: boolean; };
private _tokenizationRegistryListener: IDisposable;
constructor(modeService: IModeService, modeId: string, customization: ITokenizationCustomization, supportsNestedModes: boolean) {
this._modeService = modeService;
this._modeId = modeId;
this.customization = customization;
this.supportsNestedModes = supportsNestedModes;
this.defaults = {
enterNestedMode: !isFunction(customization.enterNestedMode),
getNestedMode: !isFunction(customization.getNestedMode),
getLeavingNestedModeData: !isFunction(customization.getLeavingNestedModeData),
};
this._embeddedModes = Object.create(null);
// Set up listening for embedded modes
let emitting = false;
this._tokenizationRegistryListener = modes.TokenizationRegistry.onDidChange((e) => {
if (emitting) {
return;
}
let isOneOfMyEmbeddedModes = this._embeddedModes[e.languageId];
if (isOneOfMyEmbeddedModes) {
emitting = true;
modes.TokenizationRegistry.fire(this._modeId);
emitting = false;
}
});
}
public dispose(): void {
this._tokenizationRegistryListener.dispose();
}
public getInitialState(): modes.IState {
return this.customization.getInitialState();
}
public tokenize(line: string, state: modes.IState, deltaOffset: number = 0, stopAtOffset: number = deltaOffset + line.length): modes.ILineTokens {
if (state.getModeId() !== this._modeId) {
return this._nestedTokenize(line, state, deltaOffset, stopAtOffset, [], []);
} else {
return this._myTokenize(line, <AbstractState>state, deltaOffset, stopAtOffset, [], []);
}
}
/**
* Precondition is: nestedModeState.getModeId() !== this._modeId
* This means we are in a nested mode when parsing starts on this line.
*/
private _nestedTokenize(buffer: string, nestedModeState: modes.IState, deltaOffset: number, stopAtOffset: number, prependTokens: Token[], prependModeTransitions: ModeTransition[]): modes.ILineTokens {
let myStateBeforeNestedMode = nestedModeState.getStateData();
let leavingNestedModeData = this._getLeavingNestedModeData(buffer, myStateBeforeNestedMode);
// Be sure to give every embedded mode the
// opportunity to leave nested mode.
// i.e. Don't go straight to the most nested mode
let stepOnceNestedState = nestedModeState;
while (stepOnceNestedState.getStateData() && stepOnceNestedState.getStateData().getModeId() !== this._modeId) {
stepOnceNestedState = stepOnceNestedState.getStateData();
}
let nestedModeId = stepOnceNestedState.getModeId();
if (!leavingNestedModeData) {
// tokenization will not leave nested mode
let result: modes.ILineTokens;
let tokenizationSupport = modes.TokenizationRegistry.get(nestedModeId);
if (tokenizationSupport) {
result = tokenizationSupport.tokenize(buffer, nestedModeState, deltaOffset, stopAtOffset);
} else {
// The nested mode doesn't have tokenization support,
// unfortunatelly this means we have to fake it
result = nullTokenize(nestedModeId, buffer, nestedModeState, deltaOffset);
}
result.tokens = prependTokens.concat(result.tokens);
result.modeTransitions = prependModeTransitions.concat(result.modeTransitions);
return result;
}
let nestedModeBuffer = leavingNestedModeData.nestedModeBuffer;
if (nestedModeBuffer.length > 0) {
// Tokenize with the nested mode
let nestedModeLineTokens: modes.ILineTokens;
let tokenizationSupport = modes.TokenizationRegistry.get(nestedModeId);
if (tokenizationSupport) {
nestedModeLineTokens = tokenizationSupport.tokenize(nestedModeBuffer, nestedModeState, deltaOffset, stopAtOffset);
} else {
// The nested mode doesn't have tokenization support,
// unfortunatelly this means we have to fake it
nestedModeLineTokens = nullTokenize(nestedModeId, nestedModeBuffer, nestedModeState, deltaOffset);
}
// Save last state of nested mode
nestedModeState = nestedModeLineTokens.endState;
// Prepend nested mode's result to our result
prependTokens = prependTokens.concat(nestedModeLineTokens.tokens);
prependModeTransitions = prependModeTransitions.concat(nestedModeLineTokens.modeTransitions);
}
let bufferAfterNestedMode = leavingNestedModeData.bufferAfterNestedMode;
let myStateAfterNestedMode = leavingNestedModeData.stateAfterNestedMode;
myStateAfterNestedMode.setStateData(myStateBeforeNestedMode.getStateData());
return this._myTokenize(bufferAfterNestedMode, myStateAfterNestedMode, deltaOffset + nestedModeBuffer.length, stopAtOffset, prependTokens, prependModeTransitions);
}
/**
* Precondition is: state.getMode() === this
* This means we are in the current mode when parsing starts on this line.
*/
private _myTokenize(buffer: string, myState: AbstractState, deltaOffset: number, stopAtOffset: number, prependTokens: Token[], prependModeTransitions: ModeTransition[]): modes.ILineTokens {
let lineStream = new LineStream(buffer);
let tokenResult: ITokenizationResult, beforeTokenizeStreamPos: number;
let previousType: string = null;
myState = myState.clone();
if (prependModeTransitions.length <= 0 || prependModeTransitions[prependModeTransitions.length - 1].modeId !== this._modeId) {
// Avoid transitioning to the same mode (this can happen in case of empty embedded modes)
prependModeTransitions.push(new ModeTransition(deltaOffset, this._modeId));
}
let maxPos = Math.min(stopAtOffset - deltaOffset, buffer.length);
while (lineStream.pos() < maxPos) {
beforeTokenizeStreamPos = lineStream.pos();
do {
tokenResult = myState.tokenize(lineStream);
if (tokenResult === null || tokenResult === undefined ||
((tokenResult.type === undefined || tokenResult.type === null) &&
(tokenResult.nextState === undefined || tokenResult.nextState === null))) {
throw new Error('Tokenizer must return a valid state');
}
if (tokenResult.nextState) {
tokenResult.nextState.setStateData(myState.getStateData());
myState = tokenResult.nextState;
}
if (lineStream.pos() <= beforeTokenizeStreamPos) {
throw new Error('Stream did not advance while tokenizing. Mode id is ' + this._modeId + ' (stuck at token type: "' + tokenResult.type + '", prepend tokens: "' + (prependTokens.map(t => t.type).join(',')) + '").');
}
} while (!tokenResult.type && tokenResult.type !== '');
if (previousType !== tokenResult.type || tokenResult.dontMergeWithPrev || previousType === null) {
prependTokens.push(new Token(beforeTokenizeStreamPos + deltaOffset, tokenResult.type));
}
previousType = tokenResult.type;
if (this.supportsNestedModes && this._enterNestedMode(myState)) {
let currentEmbeddedLevels = this._getEmbeddedLevel(myState);
if (currentEmbeddedLevels < TokenizationSupport.MAX_EMBEDDED_LEVELS) {
let nestedModeState = this._getNestedModeInitialState(myState);
if (!lineStream.eos()) {
// There is content from the embedded mode
let restOfBuffer = buffer.substr(lineStream.pos());
let result = this._nestedTokenize(restOfBuffer, nestedModeState, deltaOffset + lineStream.pos(), stopAtOffset, prependTokens, prependModeTransitions);
return result;
} else {
// Transition to the nested mode state
return {
tokens: prependTokens,
actualStopOffset: lineStream.pos() + deltaOffset,
modeTransitions: prependModeTransitions,
endState: nestedModeState
};
}
}
}
}
return {
tokens: prependTokens,
actualStopOffset: lineStream.pos() + deltaOffset,
modeTransitions: prependModeTransitions,
endState: myState
};
}
private _getEmbeddedLevel(state: modes.IState): number {
let result = -1;
while (state) {
result++;
state = state.getStateData();
}
return result;
}
private _enterNestedMode(state: AbstractState): boolean {
if (this.defaults.enterNestedMode) {
return false;
}
return this.customization.enterNestedMode(state);
}
private _getNestedMode(state: AbstractState): modes.IMode {
if (this.defaults.getNestedMode) {
return null;
}
let locator: IModeLocator = {
getMode: (mimetypeOrModeId: string): modes.IMode => {
if (!mimetypeOrModeId || !this._modeService.isRegisteredMode(mimetypeOrModeId)) {
return null;
}
let modeId = this._modeService.getModeId(mimetypeOrModeId);
let mode = this._modeService.getMode(modeId);
if (mode) {
// Re-emit tokenizationSupport change events from all modes that I ever embedded
this._embeddedModes[modeId] = true;
return mode;
}
// Fire mode loading event
this._modeService.getOrCreateMode(modeId);
this._embeddedModes[modeId] = true;
return null;
}
};
return this.customization.getNestedMode(state, locator);
}
private _getNestedModeInitialState(state: AbstractState): modes.IState {
let nestedMode = this._getNestedMode(state);
if (nestedMode) {
let tokenizationSupport = modes.TokenizationRegistry.get(nestedMode.getId());
if (tokenizationSupport) {
let nestedModeState = tokenizationSupport.getInitialState();
nestedModeState.setStateData(state);
return nestedModeState;
}
}
return new NullState(nestedMode ? nestedMode.getId() : NULL_MODE_ID, state);
}
private _getLeavingNestedModeData(line: string, state: modes.IState): ILeavingNestedModeData {
if (this.defaults.getLeavingNestedModeData) {
return null;
}
return this.customization.getLeavingNestedModeData(line, state);
}
}
src/vs/editor/common/modes/textToHtmlTokenizer.ts
浏览文件 @ 36972d22
......@@ -23,7 +23,7 @@ function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport {
return tokenizationSupport;
}
return {
getInitialState: () => new NullState(null, null),
getInitialState: () => new NullState(null),
tokenize: (buffer: string, state: IState, deltaOffset: number = 0, stopAtOffset?: number) => nullTokenize(null, buffer, state, deltaOffset, stopAtOffset)
};
}
......
src/vs/editor/common/services/modeServiceImpl.ts
浏览文件 @ 36972d22
......@@ -20,7 +20,6 @@ import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
import { LanguagesRegistry } from 'vs/editor/common/services/languagesRegistry';
import { ILanguageExtensionPoint, IValidLanguageExtensionPoint, IModeLookupResult, IModeService } from 'vs/editor/common/services/modeService';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import { AbstractState } from 'vs/editor/common/modes/abstractState';
import { Token } from 'vs/editor/common/core/token';
import { ModeTransition } from 'vs/editor/common/core/modeTransition';
......@@ -308,47 +307,33 @@ export class ModeServiceImpl implements IModeService {
export class TokenizationState2Adapter implements modes.IState {
private _modeId: string;
private _actual: modes.IState2;
private _stateData: modes.IState;
private readonly _modeId: string;
public readonly actual: modes.IState2;
constructor(modeId: string, actual: modes.IState2, stateData: modes.IState) {
constructor(modeId: string, actual: modes.IState2) {
this._modeId = modeId;
this._actual = actual;
this._stateData = stateData;
this.actual = actual;
}
public get actual(): modes.IState2 { return this._actual; }
public clone(): TokenizationState2Adapter {
return new TokenizationState2Adapter(this._modeId, this._actual.clone(), AbstractState.safeClone(this._stateData));
let actualClone = this.actual.clone();
if (actualClone === this.actual) {
return this;
}
return new TokenizationState2Adapter(this._modeId, actualClone);
}
public equals(other: modes.IState): boolean {
if (other instanceof TokenizationState2Adapter) {
if (!this._actual.equals(other._actual)) {
return false;
}
return AbstractState.safeEquals(this._stateData, other._stateData);
}
return false;
return (
other instanceof TokenizationState2Adapter
&& this._modeId === other._modeId
&& this.actual.equals(other.actual)
);
}
public getModeId(): string {
return this._modeId;
}
public tokenize(stream: any): any {
throw new Error('Unexpected tokenize call!');
}
public getStateData(): modes.IState {
return this._stateData;
}
public setStateData(stateData: modes.IState): void {
this._stateData = stateData;
}
}
export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
......@@ -362,32 +347,41 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
}
public getInitialState(): modes.IState {
return new TokenizationState2Adapter(this._modeId, this._actual.getInitialState(), null);
return new TokenizationState2Adapter(this._modeId, this._actual.getInitialState());
}
public tokenize(line: string, state: modes.IState, offsetDelta: number = 0, stopAtOffset?: number): modes.ILineTokens {
if (state instanceof TokenizationState2Adapter) {
let actualResult = this._actual.tokenize(line, state.actual);
let tokens: Token[] = [];
actualResult.tokens.forEach((t) => {
if (typeof t.scopes === 'string') {
tokens.push(new Token(t.startIndex + offsetDelta, <string>t.scopes));
} else if (Array.isArray(t.scopes) && t.scopes.length === 1) {
tokens.push(new Token(t.startIndex + offsetDelta, t.scopes[0]));
} else {
throw new Error('Only token scopes as strings or of precisely 1 length are supported at this time!');
}
});
return {
tokens: tokens,
actualStopOffset: offsetDelta + line.length,
endState: new TokenizationState2Adapter(state.getModeId(), actualResult.endState, state.getStateData()),
modeTransitions: [new ModeTransition(offsetDelta, state.getModeId())],
};
if (!(state instanceof TokenizationState2Adapter)) {
throw new Error('Unexpected state to tokenize with!');
}
throw new Error('Unexpected state to tokenize with!');
}
let actualResult = this._actual.tokenize(line, state.actual);
let tokens: Token[] = [];
actualResult.tokens.forEach((t) => {
if (typeof t.scopes === 'string') {
tokens.push(new Token(t.startIndex + offsetDelta, <string>t.scopes));
} else if (Array.isArray(t.scopes) && t.scopes.length === 1) {
tokens.push(new Token(t.startIndex + offsetDelta, t.scopes[0]));
} else {
throw new Error('Only token scopes as strings or of precisely 1 length are supported at this time!');
}
});
let endState: TokenizationState2Adapter;
// try to save an object if possible
if (actualResult.endState.equals(state.actual)) {
endState = state;
} else {
endState = new TokenizationState2Adapter(state.getModeId(), actualResult.endState);
}
return {
tokens: tokens,
actualStopOffset: offsetDelta + line.length,
endState: endState,
modeTransitions: [new ModeTransition(offsetDelta, state.getModeId())],
};
}
}
export class MainThreadModeServiceImpl extends ModeServiceImpl {
......
src/vs/editor/node/textMate/TMState.ts
浏览文件 @ 36972d22
......@@ -5,64 +5,40 @@
'use strict';
import { IState } from 'vs/editor/common/modes';
import { AbstractState } from 'vs/editor/common/modes/abstractState';
import { StackElement } from 'vscode-textmate';
export class TMState implements IState {
private _modeId: string;
private _parentEmbedderState: IState;
private _ruleStack: StackElement;
private readonly _modeId: string;
public readonly ruleStack: StackElement;
constructor(modeId: string, parentEmbedderState: IState, ruleStack: StackElement) {
constructor(modeId: string, ruleStack: StackElement) {
this._modeId = modeId;
this._parentEmbedderState = parentEmbedderState;
this._ruleStack = ruleStack;
this.ruleStack = ruleStack;
}
public clone(): TMState {
let parentEmbedderStateClone = AbstractState.safeClone(this._parentEmbedderState);
return new TMState(this._modeId, parentEmbedderStateClone, this._ruleStack);
return this;
}
public equals(other: IState): boolean {
if (!other || !(other instanceof TMState)) {
return false;
}
var otherState = <TMState>other;
// Equals on `_parentEmbedderState`
if (!AbstractState.safeEquals(this._parentEmbedderState, otherState._parentEmbedderState)) {
if (this._modeId !== other._modeId) {
return false;
}
// Equals on `_ruleStack`
if (this._ruleStack === null && otherState._ruleStack === null) {
if (this.ruleStack === null && other.ruleStack === null) {
return true;
}
if (this._ruleStack === null || otherState._ruleStack === null) {
if (this.ruleStack === null || other.ruleStack === null) {
return false;
}
return this._ruleStack.equals(otherState._ruleStack);
return this.ruleStack.equals(other.ruleStack);
}
public getModeId(): string {
return this._modeId;
}
public getStateData(): IState {
return this._parentEmbedderState;
}
public setStateData(state: IState): void {
this._parentEmbedderState = state;
}
public getRuleStack(): StackElement {
return this._ruleStack;
}
public setRuleStack(ruleStack: StackElement): void {
this._ruleStack = ruleStack;
}
}
\ No newline at end of file
}
src/vs/editor/node/textMate/TMSyntax.ts
浏览文件 @ 36972d22
......@@ -281,7 +281,7 @@ export class MainProcessTextMateSyntax {
function createTokenizationSupport(languageRegistration: TMLanguageRegistration, modeId: string, grammar: IGrammar): ITokenizationSupport {
var tokenizer = new Tokenizer(languageRegistration, modeId, grammar);
return {
getInitialState: () => new TMState(modeId, null, null),
getInitialState: () => new TMState(modeId, null),
tokenize: (line, state, offsetDelta?, stopAtOffset?) => tokenizer.tokenize(line, <TMState>state, offsetDelta, stopAtOffset)
};
}
......@@ -457,7 +457,7 @@ class Tokenizer {
public tokenize(line: string, state: TMState, offsetDelta: number = 0, stopAtOffset?: number): ILineTokens {
// Do not attempt to tokenize if a line has over 20k
// or if the rule stack contains more than 100 rules (indicator of broken grammar that forgets to pop rules)
if (line.length >= 20000 || depth(state.getRuleStack()) > 100) {
if (line.length >= 20000 || depth(state.ruleStack) > 100) {
return new RawLineTokens(
[new Token(offsetDelta, '')],
[new ModeTransition(offsetDelta, state.getModeId())],
......@@ -465,11 +465,17 @@ class Tokenizer {
state
);
}
let freshState = state.clone();
let textMateResult = this._grammar.tokenizeLine(line, freshState.getRuleStack());
freshState.setRuleStack(textMateResult.ruleStack);
let textMateResult = this._grammar.tokenizeLine(line, state.ruleStack);
return decodeTextMateTokens(line, offsetDelta, this._decodeMap, textMateResult.tokens, freshState);
let endState: TMState;
// try to save an object if possible
if (textMateResult.ruleStack.equals(state.ruleStack)) {
endState = state;
} else {
endState = new TMState(state.getModeId(), textMateResult.ruleStack);
}
return decodeTextMateTokens(line, offsetDelta, this._decodeMap, textMateResult.tokens, endState);
}
}
......
src/vs/editor/test/common/model/model.modes.test.ts
浏览文件 @ 36972d22
......@@ -32,8 +32,6 @@ suite('Editor Model - Model Modes 1', () => {
clone(): modes.IState { return this; }
equals(other: modes.IState): boolean { return this === other; }
getModeId(): string { return LANGUAGE_ID; }
getStateData(): modes.IState { throw new Error('Not implemented'); }
setStateData(state: modes.IState): void { throw new Error('Not implemented'); }
}
modes.TokenizationRegistry.register(LANGUAGE_ID, {
......@@ -182,9 +180,6 @@ suite('Editor Model - Model Modes 2', () => {
getModeId(): string {
return LANGUAGE_ID;
}
getStateData(): modes.IState { throw new Error('Not implemented'); }
setStateData(state: modes.IState): void { throw new Error('Not implemented'); }
}
modes.TokenizationRegistry.register(LANGUAGE_ID, {
......@@ -309,8 +304,6 @@ suite('Editor Model - Token Iterator', () => {
clone(): modes.IState { return this; }
equals(other: modes.IState): boolean { return this === other; }
getModeId(): string { return LANGUAGE_ID; }
getStateData(): modes.IState { throw new Error('Not implemented'); }
setStateData(state: modes.IState): void { throw new Error('Not implemented'); }
}
modes.TokenizationRegistry.register(LANGUAGE_ID, {
......
src/vs/editor/test/common/model/textModelWithTokens.test.ts
浏览文件 @ 36972d22
......@@ -264,8 +264,6 @@ suite('TextModelWithTokens regression tests', () => {
clone(): IState { return this; }
equals(other: IState): boolean { return true; }
getModeId(): string { throw new Error('Not implemented'); }
getStateData(): IState { throw new Error('Not implemented'); }
setStateData(state: IState): void { throw new Error('Not implemented'); }
}
class IndicisiveMode extends MockMode {
constructor() {
......
src/vs/editor/test/common/modes/lineStream.test.ts 已删除 100644 → 0
浏览文件 @ 92dfc35a
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as assert from 'assert';
import { LineStream } from 'vs/editor/common/modes/lineStream';
suite('Editor Modes - LineStream', () => {
test('corner cases', () => {
let noTokens = (lineStream) => {
assert.equal(lineStream.pos(), 0);
assert.ok(lineStream.eos());
};
noTokens(new LineStream(''));
});
test('advanceToEOS', () => {
var lineStream = new LineStream(' var foo =bar("foo"); //x ');
assert.equal(lineStream.pos(), 0);
lineStream.advanceToEOS();
assert.ok(lineStream.eos(), 'Stream finished');
});
test('peek', () => {
var lineStream = new LineStream('albert, bart, charlie, damon, erich');
assert.equal(lineStream.peek(), 'a');
lineStream.advance(1);
assert.equal(lineStream.peek(), 'l');
lineStream.advanceToEOS();
assert.throws(() => { lineStream.peek(); });
});
});
src/vs/editor/test/common/modes/tokenization.test.ts 已删除 100644 → 0
浏览文件 @ 92dfc35a
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as assert from 'assert';
import * as modes from 'vs/editor/common/modes';
import { AbstractState, ITokenizationResult } from 'vs/editor/common/modes/abstractState';
import { createScopedLineTokens } from 'vs/editor/common/modes/supports';
import { IModeLocator, ILeavingNestedModeData, TokenizationSupport } from 'vs/editor/common/modes/supports/tokenizationSupport';
import { createFakeLineTokens } from 'vs/editor/test/common/modesTestUtils';
import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
import { ModeTransition } from 'vs/editor/common/core/modeTransition';
import { Token } from 'vs/editor/common/core/token';
import { LineStream } from 'vs/editor/common/modes/lineStream';
export interface IModeSwitchingDescriptor {
[character: string]: {
endCharacter: string;
mode: modes.IMode;
};
}
export class StateMemorizingLastWord extends AbstractState {
public lastWord: string;
private descriptor: IModeSwitchingDescriptor;
constructor(modeId: string, descriptor: IModeSwitchingDescriptor, lastWord: string) {
super(modeId);
this.lastWord = lastWord;
this.descriptor = descriptor;
}
public makeClone(): AbstractState {
return new StateMemorizingLastWord(this.getModeId(), this.descriptor, this.lastWord);
}
public tokenize(stream: LineStream): ITokenizationResult {
let contents = stream.advanceToEOS();
stream.goBack(contents.length);
let m = contents.match(/^([\t \u00a0]+)/);
if (m) {
stream.advance(m[0].length);
return { type: '' };
}
m = contents.match(/^([\[\]\{\}\(\)])/);
let word: string;
if (m) {
stream.advance(m[0].length);
word = m[1];
} else {
m = contents.match(/([a-zA-Z]+)/);
stream.advance(m[0].length);
word = m[1];
}
return {
type: this.getModeId() + '.' + word,
nextState: new StateMemorizingLastWord(this.getModeId(), this.descriptor, word)
};
}
}
export class SwitchingMode extends MockMode {
private _switchingModeDescriptor: IModeSwitchingDescriptor;
constructor(id: string, descriptor: IModeSwitchingDescriptor) {
super(id);
this._switchingModeDescriptor = descriptor;
modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), this, true));
}
public getInitialState(): AbstractState {
return new StateMemorizingLastWord(this.getId(), this._switchingModeDescriptor, null);
}
public enterNestedMode(state: modes.IState): boolean {
var s = <StateMemorizingLastWord>state;
if (this._switchingModeDescriptor.hasOwnProperty(s.lastWord)) {
return true;
}
}
public getNestedMode(state: modes.IState, locator: IModeLocator): modes.IMode {
var s = <StateMemorizingLastWord>state;
return this._switchingModeDescriptor[s.lastWord].mode;
}
public getLeavingNestedModeData(line: string, state: modes.IState): ILeavingNestedModeData {
var s = <StateMemorizingLastWord>state;
var endChar = this._switchingModeDescriptor[s.lastWord].endCharacter;
var endCharPosition = line.indexOf(endChar);
if (endCharPosition >= 0) {
return {
nestedModeBuffer: line.substring(0, endCharPosition),
bufferAfterNestedMode: line.substring(endCharPosition),
stateAfterNestedMode: new StateMemorizingLastWord(this.getId(), this._switchingModeDescriptor, null)
};
}
return null;
}
}
interface ITestToken {
startIndex: number;
type: string;
}
function assertTokens(actual: Token[], expected: ITestToken[], message?: string) {
assert.equal(actual.length, expected.length, 'Lengths mismatch');
for (var i = 0; i < expected.length; i++) {
assert.equal(actual[i].startIndex, expected[i].startIndex, 'startIndex mismatch');
assert.equal(actual[i].type, expected[i].type, 'type mismatch');
}
};
interface ITestModeTransition {
startIndex: number;
id: string;
}
function assertModeTransitions(actual: ModeTransition[], expected: ITestModeTransition[], message?: string) {
var massagedActual: ITestModeTransition[] = [];
for (var i = 0; i < actual.length; i++) {
massagedActual.push({
startIndex: actual[i].startIndex,
id: actual[i].modeId
});
}
assert.deepEqual(massagedActual, expected, message);
};
let switchingMode = (function () {
var modeB = new SwitchingMode('B', {});
var modeC = new SwitchingMode('C', {});
var modeD = new SwitchingMode('D', {
'(': {
endCharacter: ')',
mode: modeB
}
});
var modeA = new SwitchingMode('A', {
'(': {
endCharacter: ')',
mode: modeB
},
'[': {
endCharacter: ']',
mode: modeC
},
'{': {
endCharacter: '}',
mode: modeD
}
});
return modeA;
})();
function switchingModeTokenize(line: string, state: modes.IState = null) {
let tokenizationSupport = modes.TokenizationRegistry.get(switchingMode.getId());
if (state) {
return tokenizationSupport.tokenize(line, state);
} else {
return tokenizationSupport.tokenize(line, tokenizationSupport.getInitialState());
}
}
suite('Editor Modes - Tokenization', () => {
test('Syntax engine merges sequential untyped tokens', () => {
class State extends AbstractState {
constructor(modeId: string) {
super(modeId);
}
public makeClone(): AbstractState {
return new State(this.getModeId());
}
public tokenize(stream: LineStream): ITokenizationResult {
let chr = stream.peek();
stream.advance(1);
return { type: chr === '.' ? '' : 'text' };
}
}
let tokenizationSupport = new TokenizationSupport(null, 'test', {
getInitialState: () => new State('test')
}, false);
var lineTokens = tokenizationSupport.tokenize('.abc..def...gh', tokenizationSupport.getInitialState());
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: '' },
{ startIndex: 1, type: 'text' },
{ startIndex: 4, type: '' },
{ startIndex: 6, type: 'text' },
{ startIndex: 9, type: '' },
{ startIndex: 12, type: 'text' }
]);
});
test('Warmup', () => {
var lineTokens = switchingModeTokenize('abc def ghi');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'A.def' },
{ startIndex: 7, type: '' },
{ startIndex: 8, type: 'A.ghi' }
]);
assert.equal((<StateMemorizingLastWord>lineTokens.endState).lastWord, 'ghi');
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' }
]);
});
test('One embedded', () => {
var lineTokens = switchingModeTokenize('abc (def) ghi');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'A.(' },
{ startIndex: 5, type: 'B.def' },
{ startIndex: 8, type: 'A.)' },
{ startIndex: 9, type: '' },
{ startIndex: 10, type: 'A.ghi' }
]);
assert.equal((<StateMemorizingLastWord>lineTokens.endState).lastWord, 'ghi');
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' },
{ startIndex: 5, id: 'B' },
{ startIndex: 8, id: 'A' }
]);
});
test('Empty one embedded', () => {
var lineTokens = switchingModeTokenize('abc () ghi');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'A.(' },
{ startIndex: 5, type: 'A.)' },
{ startIndex: 6, type: '' },
{ startIndex: 7, type: 'A.ghi' }
]);
assert.equal((<StateMemorizingLastWord>lineTokens.endState).lastWord, 'ghi');
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' }
]);
});
test('Finish in embedded', () => {
var lineTokens = switchingModeTokenize('abc (');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'A.(' }
]);
assert.equal((<StateMemorizingLastWord>lineTokens.endState).getModeId(), 'B');
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' }
]);
});
test('One embedded over multiple lines 1', () => {
var lineTokens = switchingModeTokenize('abc (def');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'A.(' },
{ startIndex: 5, type: 'B.def' }
]);
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' },
{ startIndex: 5, id: 'B' }
]);
lineTokens = switchingModeTokenize('ghi jkl', lineTokens.endState);
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'B.ghi' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'B.jkl' }
]);
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'B' }
]);
lineTokens = switchingModeTokenize('mno)pqr', lineTokens.endState);
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'B.mno' },
{ startIndex: 3, type: 'A.)' },
{ startIndex: 4, type: 'A.pqr' }
]);
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'B' },
{ startIndex: 3, id: 'A' }
]);
});
test('One embedded over multiple lines 2 with handleEvent', () => {
var lineTokens = switchingModeTokenize('abc (def');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'A.(' },
{ startIndex: 5, type: 'B.def' }
]);
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' },
{ startIndex: 5, id: 'B' }
]);
let scopedLineTokens1 = createScopedLineTokens(createFakeLineTokens('abc (def', switchingMode.getId(), lineTokens.tokens, lineTokens.modeTransitions), 0);
assert.deepEqual(scopedLineTokens1.modeId, 'A');
assert.equal(scopedLineTokens1.getTokenCount(), 3);
assert.equal(scopedLineTokens1.getTokenStartOffset(0), 0);
assert.equal(scopedLineTokens1.getTokenStartOffset(1), 3);
assert.equal(scopedLineTokens1.getTokenStartOffset(2), 4);
assert.deepEqual(scopedLineTokens1.firstCharOffset, 0);
assert.equal(scopedLineTokens1.getLineContent(), 'abc (');
let scopedLineTokens2 = createScopedLineTokens(createFakeLineTokens('abc (def', switchingMode.getId(), lineTokens.tokens, lineTokens.modeTransitions), 6);
assert.deepEqual(scopedLineTokens2.modeId, 'B');
assert.equal(scopedLineTokens2.getTokenCount(), 1);
assert.equal(scopedLineTokens2.getTokenStartOffset(0), 0);
assert.deepEqual(scopedLineTokens2.firstCharOffset, 5);
assert.equal(scopedLineTokens2.getLineContent(), 'def');
lineTokens = switchingModeTokenize('ghi jkl', lineTokens.endState);
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'B.ghi' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'B.jkl' }
]);
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'B' }
]);
lineTokens = switchingModeTokenize(')pqr', lineTokens.endState);
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.)' },
{ startIndex: 1, type: 'A.pqr' }
]);
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' }
]);
});
test('Two embedded in breadth', () => {
var lineTokens = switchingModeTokenize('abc (def) [ghi] jkl');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'A.(' },
{ startIndex: 5, type: 'B.def' },
{ startIndex: 8, type: 'A.)' },
{ startIndex: 9, type: '' },
{ startIndex: 10, type: 'A.[' },
{ startIndex: 11, type: 'C.ghi' },
{ startIndex: 14, type: 'A.]' },
{ startIndex: 15, type: '' },
{ startIndex: 16, type: 'A.jkl' }
]);
assert.equal((<StateMemorizingLastWord>lineTokens.endState).lastWord, 'jkl');
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' },
{ startIndex: 5, id: 'B' },
{ startIndex: 8, id: 'A' },
{ startIndex: 11, id: 'C' },
{ startIndex: 14, id: 'A' }
]);
});
test('Two embedded in breadth tightly', () => {
var lineTokens = switchingModeTokenize('abc(def)[ghi]jkl');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: 'A.(' },
{ startIndex: 4, type: 'B.def' },
{ startIndex: 7, type: 'A.)' },
{ startIndex: 8, type: 'A.[' },
{ startIndex: 9, type: 'C.ghi' },
{ startIndex: 12, type: 'A.]' },
{ startIndex: 13, type: 'A.jkl' }
]);
assert.equal((<StateMemorizingLastWord>lineTokens.endState).lastWord, 'jkl');
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' },
{ startIndex: 4, id: 'B' },
{ startIndex: 7, id: 'A' },
{ startIndex: 9, id: 'C' },
{ startIndex: 12, id: 'A' }
]);
});
test('Two embedded in depth tightly', () => {
var lineTokens = switchingModeTokenize('abc{de(efg)hi}jkl');
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: 'A.abc' },
{ startIndex: 3, type: 'A.{' },
{ startIndex: 4, type: 'D.de' },
{ startIndex: 6, type: 'D.(' },
{ startIndex: 7, type: 'B.efg' },
{ startIndex: 10, type: 'D.)' },
{ startIndex: 11, type: 'D.hi' },
{ startIndex: 13, type: 'A.}' },
{ startIndex: 14, type: 'A.jkl' }
]);
assert.equal((<StateMemorizingLastWord>lineTokens.endState).lastWord, 'jkl');
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' },
{ startIndex: 4, id: 'D' },
{ startIndex: 7, id: 'B' },
{ startIndex: 10, id: 'D' },
{ startIndex: 13, id: 'A' }
]);
});
});
src/vs/editor/test/node/textMate/TMState.test.ts
浏览文件 @ 36972d22
......@@ -9,8 +9,8 @@ import { TMState } from 'vs/editor/node/textMate/TMState';
suite('Editor Modes - TMState', () => {
test('Bug #16982: Cannot read property \'length\' of null', () => {
var s1 = new TMState(null, null, null);
var s2 = new TMState(null, null, null);
var s1 = new TMState(null, null);
var s2 = new TMState(null, null);
assert.equal(s1.equals(s2), true);
});
});
src/vs/editor/test/node/textMate/TMSyntax.test.ts
浏览文件 @ 36972d22
......@@ -101,7 +101,7 @@ suite('TextMate.decodeTextMateTokens', () => {
{ startIndex: 41, endIndex: 50, scopes: ['source.html', 'script.tag.close'] },
{ startIndex: 50, endIndex: 54, scopes: ['source.html'] },
],
new TMState('html', null, null)
new TMState('html', null)
);
let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
......@@ -389,7 +389,7 @@ suite('TextMate.decodeTextMateTokens', () => {
for (let i = 0, len = tests.length; i < len; i++) {
let test = tests[i];
let actual = decodeTextMateTokens(test.line, 0, decodeMap, test.tmTokens, new TMState('html', null, null));
let actual = decodeTextMateTokens(test.line, 0, decodeMap, test.tmTokens, new TMState('html', null));
let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; });
let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
......@@ -828,7 +828,7 @@ suite('TextMate.decodeTextMateTokens', () => {
for (let i = 0, len = tests.length; i < len; i++) {
let test = tests[i];
let actual = decodeTextMateTokens(test.line, 0, decodeMap, test.tmTokens, new TMState('html', null, null));
let actual = decodeTextMateTokens(test.line, 0, decodeMap, test.tmTokens, new TMState('html', null));
let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; });
let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
......@@ -896,7 +896,7 @@ suite('TextMate.decodeTextMateTokens', () => {
for (let i = 0, len = tests.length; i < len; i++) {
let test = tests[i];
let actual = decodeTextMateTokens(test.line, 0, decodeMap, test.tmTokens, new TMState('scss', null, null));
let actual = decodeTextMateTokens(test.line, 0, decodeMap, test.tmTokens, new TMState('scss', null));
let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; });
let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或