Skip to content

V
vscode

xxadev / vscode
与 Fork 源项目一致

从无法访问的项目Fork

通知 2
Star 0
Fork 0
V
vscode

体验新版 GitCode，发现更多精彩内容 >>

提交 9e05f824 编写于 作者: A Alex Dima
浏览文件
操作

Remove unused methods from LineStream

上级 d65cbf44
隐藏空白更改
内联 并排
Showing with 42 addition and 842 deletion
src/vs/editor/common/modes/lineStream.ts
浏览文件 @ 9e05f824
......@@ -4,33 +4,6 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {CharacterClassifier} from 'vs/editor/common/core/characterClassifier';
class CharacterSet {
private static _CACHE:{ [key:string]:CharacterSet; } = {}; // TODO@Alex unbounded cache
public static getOrCreate(source:string): CharacterSet {
if (!CharacterSet._CACHE.hasOwnProperty(source)) {
CharacterSet._CACHE[source] = new CharacterSet(source);
}
return CharacterSet._CACHE[source];
}
private _classifier: CharacterClassifier<boolean>;
constructor(source:string) {
this._classifier = new CharacterClassifier<boolean>(false);
for (let i = 0, len = source.length; i < len; i++) {
this._classifier.set(source.charCodeAt(i), true);
}
}
public contains(charCode:number): boolean {
return this._classifier.get(charCode);
}
}
/**
* A LineStream is a character & token stream abstraction over a line of text. It
* is never multi-line. The stream can be navigated character by character, or
......@@ -42,23 +15,11 @@ export class LineStream {
private _source:string;
private _sourceLength:number;
private _pos:number;
private _whitespace:string;
private _whitespaceArr:CharacterSet;
private _separators:string;
private _separatorsArr:CharacterSet;
private _tokenStart:number;
private _tokenEnd:number;
constructor(source:string) {
this._source = source;
this._sourceLength = source.length;
this._pos = 0;
this._whitespace = '\t \u00a0';
this._whitespaceArr = CharacterSet.getOrCreate(this._whitespace);
this._separators = '';
this._separatorsArr = CharacterSet.getOrCreate(this._separators);
this._tokenStart = -1;
this._tokenEnd = -1;
}
/**
......@@ -86,61 +47,14 @@ export class LineStream {
return this._source[this._pos];
}
/**
* Returns the next character in the stream, and advances it by one character.
*/
public next():string {
// Check EOS
if (this._pos >= this._sourceLength) {
throw new Error('Stream is at the end');
}
// Reset peeked token
this._tokenStart = -1;
this._tokenEnd = -1;
return this._source[this._pos++];
}
/**
* Advances the stream by one character.
*/
public next2(): void {
// Check EOS
if (this._pos >= this._sourceLength) {
throw new Error('Stream is at the end');
}
// Reset peeked token
this._tokenStart = -1;
this._tokenEnd = -1;
this._pos++;
}
/**
* Advances the stream by `n` characters.
*/
public advance(n: number): string {
if (n === 0) {
return '';
}
const oldPos = this._pos;
this._pos += n;
// Reset peeked token
this._tokenStart = -1;
this._tokenEnd = -1;
return this._source.substring(oldPos, this._pos);
}
private _advance2(n: number): number {
public advance(n: number): number {
if (n === 0) {
return n;
}
this._pos += n;
// Reset peeked token
this._tokenStart = -1;
this._tokenEnd = -1;
return n;
}
......@@ -150,7 +64,6 @@ export class LineStream {
public advanceToEOS():string {
const oldPos = this._pos;
this._pos = this._sourceLength;
this.resetPeekedToken();
return this._source.substring(oldPos, this._pos);
}
......@@ -159,410 +72,5 @@ export class LineStream {
*/
public goBack(n:number) {
this._pos -= n;
this.resetPeekedToken();
}
private createPeeker(condition:RegExp|string):()=>number {
if (condition instanceof RegExp) {
return () => {
let result = condition.exec(this._source.substr(this._pos));
if (result === null) {
return 0;
} else if (result.index !== 0) {
throw new Error('Regular expression must begin with the character "^"');
}
return result[0].length;
};
} else if ((typeof condition === 'string') && condition) {
return () => {
const len = condition.length;
let match = (this._pos + len <= this._sourceLength);
for (let i = 0; match && i < len; i++) {
match = this._source.charCodeAt(this._pos + i) === condition.charCodeAt(i);
}
return (match ? len : 0);
};
}
throw new Error('Condition must be either a regular expression, function or a non-empty string');
}
// --- BEGIN `_advanceIfStringCaseInsensitive`
private _advanceIfStringCaseInsensitive(condition:string): number {
const oldPos = this._pos;
const source = this._source;
const len = condition.length;
if (len < 1 || oldPos + len > this._sourceLength) {
return 0;
}
for (let i = 0; i < len; i++) {
if (source.charAt(oldPos + i).toLowerCase() !== condition.charAt(i).toLowerCase()) {
return 0;
}
}
return len;
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public advanceIfStringCaseInsensitive(condition: string): string {
return this.advance(this._advanceIfStringCaseInsensitive(condition));
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public advanceIfStringCaseInsensitive2(condition: string): number {
return this._advance2(this._advanceIfStringCaseInsensitive(condition));
}
// --- END
// --- BEGIN `advanceIfString`
private _advanceIfString(condition: string): number {
const oldPos = this._pos;
const source = this._source;
const len = condition.length;
if (len < 1 || oldPos + len > this._sourceLength) {
return 0;
}
for (let i = 0; i < len; i++) {
if (source.charCodeAt(oldPos + i) !== condition.charCodeAt(i)) {
return 0;
}
}
return len;
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public advanceIfString(condition:string): string {
return this.advance(this._advanceIfString(condition));
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public advanceIfString2(condition: string): number {
return this._advance2(this._advanceIfString(condition));
}
// --- END
// --- BEGIN `advanceIfString`
private _advanceIfCharCode(charCode:number): number {
if (this._pos < this._sourceLength && this._source.charCodeAt(this._pos) === charCode) {
return 1;
}
return 0;
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public advanceIfCharCode(charCode: number): string {
return this.advance(this._advanceIfCharCode(charCode));
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public advanceIfCharCode2(charCode: number): number {
return this._advance2(this._advanceIfCharCode(charCode));
}
// --- END
// --- BEGIN `advanceIfRegExp`
private _advanceIfRegExp(condition:RegExp): number {
if (this._pos >= this._sourceLength) {
return 0;
}
if (!condition.test(this._source.substr(this._pos))) {
return 0;
}
return RegExp.lastMatch.length;
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public advanceIfRegExp(condition: RegExp): string {
return this.advance(this._advanceIfRegExp(condition));
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public advanceIfRegExp2(condition: RegExp): number {
return this._advance2(this._advanceIfRegExp(condition));
}
// --- END
private advanceLoop(condition:RegExp|string, isWhile:boolean, including:boolean):string {
if (this.eos()) {
return '';
}
const peeker = this.createPeeker(condition);
const oldPos = this._pos;
let n = 0;
let f = null;
if (isWhile) {
f = (n) => {
return n > 0;
};
} else {
f = (n) => {
return n === 0;
};
}
while (!this.eos() && f(n = peeker())) {
if (n > 0) {
this.advance(n);
} else {
this.next();
}
}
if (including && !this.eos()) {
this.advance(n);
}
return this._source.substring(oldPos, this._pos);
}
/**
* Advances the stream while the next characters validate a condition. Check #advanceIf for
* details on the possible types for condition.
*/
public advanceWhile(condition:RegExp|string):string {
return this.advanceLoop(condition, true, false);
}
/**
* Advances the stream until the some characters validate a condition. Check #advanceIf for
* details on the possible types for condition. The `including` boolean value indicates
* whether the stream will advance the characters that matched the condition as well, or not.
*/
public advanceUntil(condition:RegExp|string, including:boolean):string {
return this.advanceLoop(condition, false, including);
}
// --- BEGIN `advanceUntilString`
private _advanceUntilString(condition: string, including: boolean): number {
if (this.eos() || condition.length === 0) {
return 0;
}
const oldPos = this._pos;
const index = this._source.indexOf(condition, oldPos);
if (index === -1) {
// String was not found => advanced to `eos`
return (this._sourceLength - oldPos);
}
if (including) {
// String was found => advance to include `condition`
return (index + condition.length - oldPos);
}
// String was found => advance right before `condition`
return (index - oldPos);
}
public advanceUntilString(condition: string, including: boolean): string {
return this.advance(this._advanceUntilString(condition, including));
}
public advanceUntilString2(condition: string, including: boolean): number {
return this._advance2(this._advanceUntilString(condition, including));
}
// --- END
private resetPeekedToken() {
this._tokenStart = -1;
this._tokenEnd = -1;
}
/**
* The token rules define how consecutive characters should be put together as a token,
* or separated into two different tokens. They are given through a separator characters
* string and a whitespace characters string. A separator is always one token. Consecutive
* whitespace is always one token. Everything in between these two token types, is also a token.
*
* EXAMPLE: stream.setTokenRules("+-", " ");
* Setting these token rules defines the tokens for the string "123+456 - 7" as being
* ["123", "+", "456", " ", "-", " ", "7"]
*/
public setTokenRules(separators:string, whitespace:string):void {
if (this._separators !== separators || this._whitespace !== whitespace) {
this._separators = separators;
this._separatorsArr = CharacterSet.getOrCreate(this._separators);
this._whitespace = whitespace;
this._whitespaceArr = CharacterSet.getOrCreate(this._whitespace);
this.resetPeekedToken();
}
}
// --- tokens
/**
* Returns the next token, given that the stream was configured with token rules.
*/
public peekToken():string {
if (this._tokenStart !== -1) {
return this._source.substring(this._tokenStart, this._tokenEnd);
}
const source = this._source;
const sourceLength = this._sourceLength;
const whitespaceArr = this._whitespaceArr;
const separatorsArr = this._separatorsArr;
let tokenStart = this._pos;
// Check EOS
if (tokenStart >= sourceLength) {
throw new Error('Stream is at the end');
}
// Skip whitespace
while (whitespaceArr.contains(source.charCodeAt(tokenStart)) && tokenStart < sourceLength) {
tokenStart++;
}
let tokenEnd = tokenStart;
// If a separator is hit, it is a token
if (separatorsArr.contains(source.charCodeAt(tokenEnd)) && tokenEnd < sourceLength) {
tokenEnd++;
} else {
// Advance until a separator or a whitespace is hit
while (!separatorsArr.contains(source.charCodeAt(tokenEnd)) && !whitespaceArr.contains(source.charCodeAt(tokenEnd)) && tokenEnd < sourceLength) {
tokenEnd++;
}
}
// Cache peeked token
this._tokenStart = tokenStart;
this._tokenEnd = tokenEnd;
return source.substring(tokenStart, tokenEnd);
}
/**
* Returns the next token, given that the stream was configured with token rules, and advances the
* stream by the exact length of the found token.
*/
public nextToken():string {
// Check EOS
if (this._pos >= this._sourceLength) {
throw new Error('Stream is at the end');
}
// Peek token if necessary
let result:string;
if (this._tokenStart === -1) {
result = this.peekToken();
} else {
result = this._source.substring(this._tokenStart, this._tokenEnd);
}
// Advance to tokenEnd
this._pos = this._tokenEnd;
// Reset peeked token
this._tokenStart = -1;
this._tokenEnd = -1;
return result;
}
// -- whitespace
/**
* Returns the next whitespace, if found. Returns an empty string otherwise.
*/
public peekWhitespace():string {
const source = this._source;
const sourceLength = this._sourceLength;
const whitespaceArr = this._whitespaceArr;
let peek = this._pos;
while (whitespaceArr.contains(source.charCodeAt(peek)) && peek < sourceLength) {
peek++;
}
return source.substring(this._pos, peek);
}
// --- BEGIN `skipWhitespace`
private _skipWhitespace(): number {
const source = this._source;
const sourceLength = this._sourceLength;
const whitespaceArr = this._whitespaceArr;
const oldPos = this._pos;
let peek = this._pos;
while (whitespaceArr.contains(source.charCodeAt(peek)) && peek < sourceLength) {
peek++;
}
return (peek - oldPos);
}
/**
* Returns the next whitespace, if found, and advances the stream by the exact length of the found
* whitespace. Returns an empty string otherwise.
*/
public skipWhitespace(): string {
return this.advance(this._skipWhitespace());
}
/**
* Returns the next whitespace, if found, and advances the stream by the exact length of the found
* whitespace. Returns an empty string otherwise.
*/
public skipWhitespace2(): number {
return this._advance2(this._skipWhitespace());
}
// --- END
}
src/vs/editor/test/common/model/model.modes.test.ts
浏览文件 @ 9e05f824
......@@ -40,7 +40,9 @@ suite('Editor Model - Model Modes 1', () => {
return this === other;
}
public tokenize(stream:LineStream): ITokenizationResult {
calledState.calledFor.push(stream.next());
let chr = stream.peek();
stream.advance(1);
calledState.calledFor.push(chr);
stream.advanceToEOS();
return { type: '' };
}
......@@ -190,10 +192,7 @@ suite('Editor Model - Model Modes 2', () => {
}
public tokenize(stream:LineStream):ITokenizationResult {
var line= '';
while (!stream.eos()) {
line+= stream.next();
}
var line= stream.advanceToEOS();
this.prevLineContent= line;
return { type: '' };
}
......@@ -329,7 +328,9 @@ suite('Editor Model - Token Iterator', () => {
public tokenize(stream:LineStream):ITokenizationResult {
var ndash = this.n, value = '';
while(!stream.eos() && ndash > 0) {
value += stream.next();
let chr = stream.peek();
stream.advance(1);
value += chr;
ndash--;
}
return { type: 'n-' + (this.n - ndash) + '-' + value };
......
src/vs/editor/test/common/modes/lineStream.test.ts
浏览文件 @ 9e05f824
......@@ -8,361 +8,34 @@ import {LineStream} from 'vs/editor/common/modes/lineStream';
suite('Editor Modes - LineStream', () => {
test('advanceIf - regex', () => {
var lineStream = new LineStream('...xxx...x.');
assert.equal(lineStream.advanceIfRegExp(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceIfRegExp(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceIfRegExp(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceIfRegExp(/^x/), 'x');
assert.equal(lineStream.advanceIfRegExp(/^x/), 'x');
assert.equal(lineStream.advanceIfRegExp(/^x/), 'x');
assert.equal(lineStream.advanceIfRegExp(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceIfRegExp(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceIfRegExp(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceIfRegExp(/^x/), 'x');
assert.equal(lineStream.advanceIfRegExp(/^x/), '');
lineStream.next();
assert.ok(lineStream.eos());
});
test('advanceWhile - regex', () => {
var lineStream = new LineStream('...xxx...x.');
assert.equal(lineStream.advanceWhile(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceWhile(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceWhile(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceWhile(/^x/), 'xxx');
assert.equal(lineStream.advanceWhile(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceWhile(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceWhile(/^x/), '');
lineStream.next();
assert.equal(lineStream.advanceWhile(/^x/), 'x');
assert.equal(lineStream.advanceWhile(/^x/), '');
lineStream.next();
assert.ok(lineStream.eos());
});
test('advanceUntil - regex', () => {
var lineStream = new LineStream('...x..xx..x');
assert.equal(lineStream.advanceUntil(/^x/, false), '...');
assert.equal(lineStream.advanceUntil(/^x/, false), '');
lineStream.next();
assert.equal(lineStream.advanceUntil(/^x/, false), '..');
assert.equal(lineStream.advanceUntil(/^x/, false), '');
lineStream.next();
assert.equal(lineStream.advanceUntil(/^x/, false), '');
lineStream.next();
assert.equal(lineStream.advanceUntil(/^x/, false), '..');
assert.equal(lineStream.advanceUntil(/^x/, false), '');
lineStream.next();
assert.ok(lineStream.eos());
});
test('advanceUntil - regex (including)', () => {
var lineStream = new LineStream('...x..xx..x');
assert.equal(lineStream.advanceUntil(/^x/, true), '...x');
assert.equal(lineStream.advanceUntil(/^x/, true), '..x');
assert.equal(lineStream.advanceUntil(/^x/, true), 'x');
assert.equal(lineStream.advanceUntil(/^x/, true), '..x');
assert.ok(lineStream.eos());
});
test('advanceIf - string', () => {
var lineStream = new LineStream('...abcabcabc...abc.');
assert.equal(lineStream.advanceIfString('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceIfString('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceIfString('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceIfString('abc'), 'abc');
assert.equal(lineStream.advanceIfString('abc'), 'abc');
assert.equal(lineStream.advanceIfString('abc'), 'abc');
assert.equal(lineStream.advanceIfString('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceIfString('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceIfString('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceIfString('abc'), 'abc');
assert.equal(lineStream.advanceIfString('abc'), '');
lineStream.next();
assert.ok(lineStream.eos());
});
test('advanceWhile - string', () => {
var lineStream = new LineStream('...abcabcabc...abc.');
assert.equal(lineStream.advanceWhile('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceWhile('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceWhile('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceWhile('abc'), 'abcabcabc');
assert.equal(lineStream.advanceWhile('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceWhile('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceWhile('abc'), '');
lineStream.next();
assert.equal(lineStream.advanceWhile('abc'), 'abc');
assert.equal(lineStream.advanceWhile('abc'), '');
lineStream.next();
assert.ok(lineStream.eos());
});
test('advanceUntil - string', () => {
var lineStream = new LineStream('...abc..ab..abc..bc');
assert.equal(lineStream.advanceUntil('abc', false), '...');
assert.equal(lineStream.advanceUntil('abc', false), '');
lineStream.next();
assert.equal(lineStream.advanceUntil('abc', false), 'bc..ab..');
assert.equal(lineStream.advanceUntil('abc', false), '');
lineStream.next();
assert.equal(lineStream.advanceUntil('abc', false), 'bc..bc');
assert.ok(lineStream.eos());
});
test('advanceUntil - string (including)', () => {
var lineStream = new LineStream('...abc..ab..abc..bc');
assert.equal(lineStream.advanceUntil('abc', true), '...abc');
assert.equal(lineStream.advanceUntil('abc', true), '..ab..abc');
assert.equal(lineStream.advanceUntil('abc', true), '..bc');
assert.ok(lineStream.eos());
});
test('skipWhitespace', () => {
var lineStream = new LineStream('\ta bc d \t e ');
assert.equal(lineStream.skipWhitespace(), '\t');
lineStream.next();
assert.equal(lineStream.skipWhitespace(), ' ');
lineStream.next();
lineStream.next();
assert.equal(lineStream.skipWhitespace(), ' ');
lineStream.next();
assert.equal(lineStream.skipWhitespace(), ' \t ');
lineStream.next();
assert.equal(lineStream.skipWhitespace(), ' ');
assert.ok(lineStream.eos());
});
test('peekToken', () => {
var lineStream = new LineStream('a b c edf ');
assert.equal(lineStream.peekToken(), 'a');
lineStream.next();
assert.equal(lineStream.peekToken(), 'b');
lineStream.next();
assert.equal(lineStream.peekToken(), 'b');
lineStream.next();
assert.equal(lineStream.peekToken(), 'c');
lineStream.next();
assert.equal(lineStream.peekToken(), 'c');
lineStream.next();
assert.equal(lineStream.peekToken(), 'c');
lineStream.next();
assert.equal(lineStream.peekToken(), 'edf');
lineStream.next();
assert.equal(lineStream.peekToken(), 'edf');
lineStream.next();
lineStream.next();
lineStream.next();
lineStream.next();
assert.throws(() => { lineStream.peekToken(); });
assert.ok(lineStream.eos());
});
test('nextToken', () => {
var lineStream = new LineStream('a b c edf ');
assert.equal(lineStream.nextToken(), 'a');
assert.equal(lineStream.nextToken(), 'b');
assert.equal(lineStream.nextToken(), 'c');
assert.equal(lineStream.nextToken(), 'edf');
assert.equal(lineStream.nextToken(), '');
assert.throws(() => { lineStream.nextToken(); });
assert.ok(lineStream.eos());
});
function newTokenStream(source, separators, whitespace) {
var lineStream = new LineStream(source);
lineStream.setTokenRules(separators, whitespace);
return lineStream;
}
function checkPos(lineStream,pos) {
assert.equal(lineStream.pos(), pos);
}
function check(lineStream, pos, token) {
checkPos(lineStream, pos);
assert.equal(lineStream.nextToken(), token);
}
test('corner cases', () => {
var input, lineStream;
var noTokens = (lineStream) => {
let noTokens = (lineStream) => {
assert.equal(lineStream.pos(), 0);
assert.ok(lineStream.eos());
};
noTokens(newTokenStream('', '', ''));
noTokens(newTokenStream('', '', 'x'));
noTokens(newTokenStream('', 'x', ''));
noTokens(newTokenStream('', 'x', 'x'));
input = '.....';
lineStream = newTokenStream(input, '.', '');
for (var i = 0; i < input.length; i++) {
check(lineStream, i, '.');
}
input = ' . . . . .';
lineStream = newTokenStream(input, '.', ' ');
for (var i = 0; i < input.length / 2; i++) {
check(lineStream, (i * 2) , '.');
}
input = '. . . . . ';
lineStream = newTokenStream(input, '.', ' ');
for (var i = 0; i < input.length / 2; i++) {
check(lineStream, i === 0 ? 0 : (i * 2) - 1, '.');
}
noTokens(new LineStream(''));
});
test('javascript assign', () => {
var lineStream = newTokenStream(' var foo =bar("foo"); //x ', '+-*/%&|^~!=<>(){}[]\'"\\/?;,', '\t ');
test('advanceToEOS', () => {
var lineStream = new LineStream(' var foo =bar("foo"); //x ');
assert.equal(lineStream.pos(), 0);
assert.equal(lineStream.peekToken(), 'var');
check(lineStream, 0, 'var');
check(lineStream, 5, 'foo');
check(lineStream, 9, '=');
check(lineStream, 11, 'bar');
check(lineStream, 14, '(');
check(lineStream, 15, '"');
check(lineStream, 16, 'foo');
check(lineStream, 19, '"');
check(lineStream, 20, ')');
check(lineStream, 21, ';');
check(lineStream, 22, '/');
check(lineStream, 24, '/');
check(lineStream, 25, 'x');
checkPos(lineStream, 26);
lineStream.skipWhitespace();
assert.ok(lineStream.eos(), 'Stream finished');
});
test('javascript strings', () => {
var lineStream = newTokenStream('x = " my \\"string\\" ";', '=()\\";/', '\t ');
check(lineStream, 0, 'x');
check(lineStream, 1, '=');
check(lineStream, 3, '"');
check(lineStream, 5, 'my');
check(lineStream, 9, '\\');
check(lineStream, 11, '"');
check(lineStream, 12, 'string');
check(lineStream, 18, '\\');
check(lineStream, 19, '"');
check(lineStream, 20, '"');
check(lineStream, 22, ';');
lineStream.advanceToEOS();
assert.ok(lineStream.eos(), 'Stream finished');
});
test('peek', () => {
var lineStream = newTokenStream('albert, bart, charlie, damon, erich', ',', ' ');
var lineStream = new LineStream('albert, bart, charlie, damon, erich');
assert.equal(lineStream.peekToken(), 'albert');
assert.equal(lineStream.peek(), 'a');
lineStream.advance(1);
assert.equal(lineStream.nextToken(), 'albert');
assert.equal(lineStream.nextToken(), ',');
assert.equal(lineStream.peekToken(), 'bart');
assert.equal(lineStream.peek(), ' ');
assert.equal(lineStream.nextToken(), 'bart');
assert.equal(lineStream.peekToken(), ',');
assert.equal(lineStream.peek(), ',');
assert.equal(lineStream.nextToken(), ',');
lineStream.advanceToEOS();
assert.throws(() => { lineStream.peekToken(); });
assert.throws(() => { lineStream.peek(); });
});
test('next', () => {
var lineStream = newTokenStream('albert, bart, charlie, damon, erich', ',', ' ');
assert.equal(lineStream.peekToken(), 'albert');
assert.equal(lineStream.next(), 'a');
assert.equal(lineStream.next(), 'l');
assert.equal(lineStream.next(), 'b');
assert.equal(lineStream.nextToken(), 'ert');
assert.equal(lineStream.nextToken(), ',');
assert.equal(lineStream.nextToken(), 'bart');
assert.equal(lineStream.peekToken(), ',');
assert.equal(lineStream.next(), ',');
assert.equal(lineStream.next(), ' ');
assert.equal(lineStream.next(), 'c');
assert.equal(lineStream.next(), 'h');
assert.equal(lineStream.next(), 'a');
assert.equal(lineStream.next(), 'r');
assert.equal(lineStream.next(), 'l');
assert.equal(lineStream.next(), 'i');
assert.equal(lineStream.next(), 'e');
assert.equal(lineStream.next(), ',');
assert.equal(lineStream.nextToken(), 'damon');
assert.equal(lineStream.peek(), 'l');
lineStream.advanceToEOS();
assert.throws(() => { lineStream.peekToken(); });
assert.throws(() => { lineStream.peek(); });
});
test('next & goBack', () => {
var lineStream = new LineStream('albert, bart, charlie, damon, erich');
lineStream.setTokenRules(',', ' ');
assert.equal(lineStream.peekToken(), 'albert');
assert.equal(lineStream.next(), 'a');
assert.equal(lineStream.next(), 'l');
assert.equal(lineStream.next(), 'b');
assert.equal(lineStream.nextToken(), 'ert');
lineStream.goBack(6);
assert.equal(lineStream.nextToken(), 'albert');
assert.equal(lineStream.next(), ',');
lineStream.goBack(7);
assert.equal(lineStream.nextToken(), 'albert');
assert.equal(lineStream.nextToken(), ',');
assert.equal(lineStream.next(), ' ');
assert.equal(lineStream.next(), 'b');
assert.equal(lineStream.next(), 'a');
lineStream.goBack(3);
assert.equal(lineStream.nextToken(), 'bart');
lineStream.goBack(5);
assert.equal(lineStream.next(), ' ');
lineStream.advanceToEOS();
assert.throws(() => { lineStream.peekToken(); });
assert.throws(() => { lineStream.peek(); });
});
});
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
浏览文件 @ 9e05f824
......@@ -69,7 +69,9 @@ class State extends AbstractState {
}
public tokenize(stream:LineStream):ITokenizationResult {
return { type: stream.next() === '.' ? '' : 'text' };
let chr = stream.peek();
stream.advance(1);
return { type: chr === '.' ? '' : 'text' };
}
}
......
src/vs/editor/test/common/modes/tokenization.test.ts
浏览文件 @ 9e05f824
......@@ -38,13 +38,27 @@ export class StateMemorizingLastWord extends AbstractState {
}
public tokenize(stream:LineStream):ITokenizationResult {
stream.setTokenRules('[]{}()==--', '\t \u00a0');
if (stream.skipWhitespace() !== '') {
return {
type: ''
};
let contents = stream.advanceToEOS();
stream.goBack(contents.length);
let m = contents.match(/^([\t \u00a0]+)/);
if (m) {
stream.advance(m[0].length);
return { type: '' };
}
var word = stream.nextToken();
m = contents.match(/^([\[\]\{\}\(\)])/);
let word: string;
if (m) {
stream.advance(m[0].length);
word = m[1];
} else {
m = contents.match(/([a-zA-Z]+)/);
stream.advance(m[0].length);
word = m[1];
}
return {
type: this.getModeId() + '.' + word,
nextState: new StateMemorizingLastWord(this.getModeId(), this.descriptor, word)
......@@ -169,7 +183,9 @@ suite('Editor Modes - Tokenization', () => {
}
public tokenize(stream:LineStream):ITokenizationResult {
return { type: stream.next() === '.' ? '' : 'text' };
let chr = stream.peek();
stream.advance(1);
return { type: chr === '.' ? '' : 'text' };
}
}
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或