Use directly LineStream

src/vs/editor/common/modes.ts

@@ -29,126 +29,6 @@ export interface IState {
 	setStateData(state:IState):void;
 }
 
-/**
- * An IStream is a character & token stream abstraction over a line of text. It
- *  is never multi-line. The stream can be navigated character by character, or
- *  token by token, given some token rules.
- * @internal
- */
-export interface IStream {
-
-	/**
-	 * Returns the current character position of the stream on the line.
-	 */
-	pos():number;
-
-	/**
-	 * Returns true iff the stream is at the end of the line.
-	 */
-	eos():boolean;
-
-	/**
-	 * Returns the next character in the stream.
-	 */
-	peek():string;
-
-	/**
-	 * Returns the next character in the stream, and advances it by one character.
-	 */
-	next(): string;
-	next2(): void;
-
-	/**
-	 * Advances the stream by `n` characters.
-	 */
-	advance(n:number):string;
-
-	/**
-	 * Advances the stream until the end of the line.
-	 */
-	advanceToEOS():string;
-
-	/**
-	 * Brings the stream back `n` characters.
-	 */
-	goBack(n:number):void;
-
-	/**
-	 *  Advances the stream if the next characters validate a condition. A condition can be
-	 *
-	 *      - a regular expression (always starting with ^)
-	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
-	 *
-	 *  	- a string
-	 * 			EXAMPLES: "1954", "albert"
-	 */
-	advanceIfCharCode(charCode: number): string;
-	advanceIfCharCode2(charCode:number): number;
-
-	advanceIfString(condition: string): string;
-	advanceIfString2(condition: string): number;
-
-	advanceIfStringCaseInsensitive(condition: string): string;
-	advanceIfStringCaseInsensitive2(condition: string): number;
-
-	advanceIfRegExp(condition: RegExp): string;
-	advanceIfRegExp2(condition:RegExp): number;
-
-
-	/**
-	 * Advances the stream while the next characters validate a condition. Check #advanceIf for
-	 * details on the possible types for condition.
-	 */
-	advanceWhile(condition:string):string;
-	advanceWhile(condition:RegExp):string;
-
-	/**
-	 * Advances the stream until the some characters validate a condition. Check #advanceIf for
-	 * details on the possible types for condition. The `including` boolean value indicates
-	 * whether the stream will advance the characters that matched the condition as well, or not.
-	 */
-	advanceUntil(condition: string, including: boolean): string;
-	advanceUntil(condition: RegExp, including: boolean): string;
-
-	advanceUntilString(condition: string, including: boolean): string;
-	advanceUntilString2(condition: string, including: boolean): number;
-
-	/**
-	 * The token rules define how consecutive characters should be put together as a token,
-	 * or separated into two different tokens. They are given through a separator characters
-	 * string and a whitespace characters string. A separator is always one token. Consecutive
-	 * whitespace is always one token. Everything in between these two token types, is also a token.
-	 *
-	 * 	EXAMPLE: stream.setTokenRules("+-", " ");
-	 * 	Setting these token rules defines the tokens for the string "123+456 -    7" as being
-	 * 		["123", "+", "456", " ", "-", "    ", "7"]
-	 */
-	setTokenRules(separators:string, whitespace:string):void;
-
-	/**
-	 * Returns the next token, given that the stream was configured with token rules.
-	 */
-	peekToken():string;
-
-	/**
-	 * Returns the next token, given that the stream was configured with token rules, and advances the
-	 * stream by the exact length of the found token.
-	 */
-	nextToken():string;
-
-	/**
-	 * Returns the next whitespace, if found. Returns an empty string otherwise.
-	 */
-	peekWhitespace():string;
-
-	/**
-	 * Returns the next whitespace, if found, and advances the stream by the exact length of the found
-	 * whitespace. Returns an empty string otherwise.
-	 */
-	skipWhitespace(): string;
-	skipWhitespace2(): number;
-}
-
 /**
  * @internal
  */

src/vs/editor/common/modes/abstractState.ts

@@ -4,7 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IState, IStream} from 'vs/editor/common/modes';
+import {IState} from 'vs/editor/common/modes';
+import {LineStream} from 'vs/editor/common/modes/lineStream';
 
 /**
  * @internal
@@ -61,7 +62,7 @@ export abstract class AbstractState implements IState {
 		return false;
 	}
 
-	public abstract tokenize(stream:IStream):ITokenizationResult;
+	public abstract tokenize(stream:LineStream):ITokenizationResult;
 
 	public static safeEquals(a: IState, b: IState): boolean {
 		if (a === null && b === null) {

src/vs/editor/common/modes/lineStream.ts

@@ -4,7 +4,6 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IStream} from 'vs/editor/common/modes';
 import {CharacterClassifier} from 'vs/editor/common/core/characterClassifier';
 
 class CharacterSet {
@@ -32,7 +31,13 @@ class CharacterSet {
 	}
 }
 
-export class LineStream implements IStream {
+/**
+ * A LineStream is a character & token stream abstraction over a line of text. It
+ *  is never multi-line. The stream can be navigated character by character, or
+ *  token by token, given some token rules.
+ * @internal
+ */
+export class LineStream {
 
 	private _source:string;
 	private _sourceLength:number;
@@ -56,14 +61,23 @@ export class LineStream implements IStream {
 		this._tokenEnd = -1;
 	}
 
+	/**
+	 * Returns the current character position of the stream on the line.
+	 */
 	public pos():number {
 		return this._pos;
 	}
 
+	/**
+	 * Returns true iff the stream is at the end of the line.
+	 */
 	public eos() {
 		return this._pos >= this._sourceLength;
 	}
 
+	/**
+	 * Returns the next character in the stream.
+	 */
 	public peek():string {
 		// Check EOS
 		if (this._pos >= this._sourceLength) {
@@ -72,6 +86,9 @@ export class LineStream implements IStream {
 		return this._source[this._pos];
 	}
 
+	/**
+	 * Returns the next character in the stream, and advances it by one character.
+	 */
 	public next():string {
 		// Check EOS
 		if (this._pos >= this._sourceLength) {
@@ -85,6 +102,9 @@ export class LineStream implements IStream {
 		return this._source[this._pos++];
 	}
 
+	/**
+	 * Advances the stream by one character.
+	 */
 	public next2(): void {
 		// Check EOS
 		if (this._pos >= this._sourceLength) {
@@ -98,6 +118,9 @@ export class LineStream implements IStream {
 		this._pos++;
 	}
 
+	/**
+	 * Advances the stream by `n` characters.
+	 */
 	public advance(n: number): string {
 		if (n === 0) {
 			return '';
@@ -121,6 +144,9 @@ export class LineStream implements IStream {
 		return n;
 	}
 
+	/**
+	 * Advances the stream until the end of the line.
+	 */
 	public advanceToEOS():string {
 		const oldPos = this._pos;
 		this._pos = this._sourceLength;
@@ -128,6 +154,9 @@ export class LineStream implements IStream {
 		return this._source.substring(oldPos, this._pos);
 	}
 
+	/**
+	 * Brings the stream back `n` characters.
+	 */
 	public goBack(n:number) {
 		this._pos -= n;
 		this.resetPeekedToken();
@@ -175,9 +204,27 @@ export class LineStream implements IStream {
 
 		return len;
 	}
+	/**
+	 *  Advances the stream if the next characters validate a condition. A condition can be
+	 *
+	 *      - a regular expression (always starting with ^)
+	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
+	 *
+	 *  	- a string
+	 * 			EXAMPLES: "1954", "albert"
+	 */
 	public advanceIfStringCaseInsensitive(condition: string): string {
 		return this.advance(this._advanceIfStringCaseInsensitive(condition));
 	}
+	/**
+	 *  Advances the stream if the next characters validate a condition. A condition can be
+	 *
+	 *      - a regular expression (always starting with ^)
+	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
+	 *
+	 *  	- a string
+	 * 			EXAMPLES: "1954", "albert"
+	 */
 	public advanceIfStringCaseInsensitive2(condition: string): number {
 		return this._advance2(this._advanceIfStringCaseInsensitive(condition));
 	}
@@ -201,9 +248,27 @@ export class LineStream implements IStream {
 
 		return len;
 	}
+	/**
+	 *  Advances the stream if the next characters validate a condition. A condition can be
+	 *
+	 *      - a regular expression (always starting with ^)
+	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
+	 *
+	 *  	- a string
+	 * 			EXAMPLES: "1954", "albert"
+	 */
 	public advanceIfString(condition:string): string {
 		return this.advance(this._advanceIfString(condition));
 	}
+	/**
+	 *  Advances the stream if the next characters validate a condition. A condition can be
+	 *
+	 *      - a regular expression (always starting with ^)
+	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
+	 *
+	 *  	- a string
+	 * 			EXAMPLES: "1954", "albert"
+	 */
 	public advanceIfString2(condition: string): number {
 		return this._advance2(this._advanceIfString(condition));
 	}
@@ -217,9 +282,27 @@ export class LineStream implements IStream {
 
 		return 0;
 	}
+	/**
+	 *  Advances the stream if the next characters validate a condition. A condition can be
+	 *
+	 *      - a regular expression (always starting with ^)
+	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
+	 *
+	 *  	- a string
+	 * 			EXAMPLES: "1954", "albert"
+	 */
 	public advanceIfCharCode(charCode: number): string {
 		return this.advance(this._advanceIfCharCode(charCode));
 	}
+	/**
+	 *  Advances the stream if the next characters validate a condition. A condition can be
+	 *
+	 *      - a regular expression (always starting with ^)
+	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
+	 *
+	 *  	- a string
+	 * 			EXAMPLES: "1954", "albert"
+	 */
 	public advanceIfCharCode2(charCode: number): number {
 		return this._advance2(this._advanceIfCharCode(charCode));
 	}
@@ -235,9 +318,27 @@ export class LineStream implements IStream {
 		}
 		return RegExp.lastMatch.length;
 	}
+	/**
+	 *  Advances the stream if the next characters validate a condition. A condition can be
+	 *
+	 *      - a regular expression (always starting with ^)
+	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
+	 *
+	 *  	- a string
+	 * 			EXAMPLES: "1954", "albert"
+	 */
 	public advanceIfRegExp(condition: RegExp): string {
 		return this.advance(this._advanceIfRegExp(condition));
 	}
+	/**
+	 *  Advances the stream if the next characters validate a condition. A condition can be
+	 *
+	 *      - a regular expression (always starting with ^)
+	 * 			EXAMPLES: /^\d+/, /^function|var|interface|class/
+	 *
+	 *  	- a string
+	 * 			EXAMPLES: "1954", "albert"
+	 */
 	public advanceIfRegExp2(condition: RegExp): number {
 		return this._advance2(this._advanceIfRegExp(condition));
 	}
@@ -273,10 +374,19 @@ export class LineStream implements IStream {
 		return this._source.substring(oldPos, this._pos);
 	}
 
+	/**
+	 * Advances the stream while the next characters validate a condition. Check #advanceIf for
+	 * details on the possible types for condition.
+	 */
 	public advanceWhile(condition:RegExp|string):string {
 		return this.advanceLoop(condition, true, false);
 	}
 
+	/**
+	 * Advances the stream until the some characters validate a condition. Check #advanceIf for
+	 * details on the possible types for condition. The `including` boolean value indicates
+	 * whether the stream will advance the characters that matched the condition as well, or not.
+	 */
 	public advanceUntil(condition:RegExp|string, including:boolean):string {
 		return this.advanceLoop(condition, false, including);
 	}
@@ -316,6 +426,16 @@ export class LineStream implements IStream {
 		this._tokenEnd = -1;
 	}
 
+	/**
+	 * The token rules define how consecutive characters should be put together as a token,
+	 * or separated into two different tokens. They are given through a separator characters
+	 * string and a whitespace characters string. A separator is always one token. Consecutive
+	 * whitespace is always one token. Everything in between these two token types, is also a token.
+	 *
+	 * 	EXAMPLE: stream.setTokenRules("+-", " ");
+	 * 	Setting these token rules defines the tokens for the string "123+456 -    7" as being
+	 * 		["123", "+", "456", " ", "-", "    ", "7"]
+	 */
 	public setTokenRules(separators:string, whitespace:string):void {
 		if (this._separators !== separators || this._whitespace !== whitespace) {
 			this._separators = separators;
@@ -328,6 +448,9 @@ export class LineStream implements IStream {
 
 	// --- tokens
 
+	/**
+	 * Returns the next token, given that the stream was configured with token rules.
+	 */
 	public peekToken():string {
 		if (this._tokenStart !== -1) {
 			return this._source.substring(this._tokenStart, this._tokenEnd);
@@ -368,6 +491,10 @@ export class LineStream implements IStream {
 		return source.substring(tokenStart, tokenEnd);
 	}
 
+	/**
+	 * Returns the next token, given that the stream was configured with token rules, and advances the
+	 * stream by the exact length of the found token.
+	 */
 	public nextToken():string {
 		// Check EOS
 		if (this._pos >= this._sourceLength) {
@@ -394,6 +521,9 @@ export class LineStream implements IStream {
 
 	// -- whitespace
 
+	/**
+	 * Returns the next whitespace, if found. Returns an empty string otherwise.
+	 */
 	public peekWhitespace():string {
 		const source = this._source;
 		const sourceLength = this._sourceLength;
@@ -420,9 +550,17 @@ export class LineStream implements IStream {
 
 		return (peek - oldPos);
 	}
+	/**
+	 * Returns the next whitespace, if found, and advances the stream by the exact length of the found
+	 * whitespace. Returns an empty string otherwise.
+	 */
 	public skipWhitespace(): string {
 		return this.advance(this._skipWhitespace());
 	}
+	/**
+	 * Returns the next whitespace, if found, and advances the stream by the exact length of the found
+	 * whitespace. Returns an empty string otherwise.
+	 */
 	public skipWhitespace2(): number {
 		return this._advance2(this._skipWhitespace());
 	}

src/vs/editor/common/modes/monarch/monarchLexer.ts

@@ -94,7 +94,7 @@ export class MonarchLexer extends AbstractState {
 	 * TODO: there are many optimizations possible here for the common cases
 	 * but for now I concentrated on functionality and correctness.
 	 */
-	public tokenize(stream: modes.IStream, noConsumeIsOk?: boolean): ITokenizationResult {
+	public tokenize(stream: LineStream, noConsumeIsOk?: boolean): ITokenizationResult {
 		var stackLen0 = this.stack.length;  // these are saved to check progress
 		var groupLen0 = 0;
 		var state: string = this.stack[0];  // the current state

src/vs/editor/common/modes/nullMode.ts

@@ -4,10 +4,11 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IState, IStream, ILineTokens} from 'vs/editor/common/modes';
+import {IState, ILineTokens} from 'vs/editor/common/modes';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
 import {Token} from 'vs/editor/common/core/token';
 import {ITokenizationResult} from 'vs/editor/common/modes/abstractState';
+import {LineStream} from 'vs/editor/common/modes/lineStream';
 
 export class NullState implements IState {
 
@@ -42,7 +43,7 @@ export class NullState implements IState {
 		return this.modeId;
 	}
 
-	public tokenize(stream:IStream):ITokenizationResult {
+	public tokenize(stream:LineStream):ITokenizationResult {
 		stream.advanceToEOS();
 		return { type:'' };
 	}

src/vs/editor/test/common/mocks/mockMode.ts

@@ -4,9 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IMode, IState, IStream, TokenizationRegistry} from 'vs/editor/common/modes';
+import {IMode, IState, TokenizationRegistry} from 'vs/editor/common/modes';
 import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
+import {LineStream} from 'vs/editor/common/modes/lineStream';
 
 let instanceCount = 0;
 function generateMockModeId(): string {
@@ -45,7 +46,7 @@ export class StateForMockTokenizingMode extends AbstractState {
 		return true;
 	}
 
-	public tokenize(stream:IStream):ITokenizationResult {
+	public tokenize(stream:LineStream):ITokenizationResult {
 		stream.advanceToEOS();
 		return { type: this._tokenType };
 	}

src/vs/editor/test/common/model/model.modes.test.ts

@@ -12,6 +12,7 @@ import {Model} from 'vs/editor/common/model/model';
 import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 import * as modes from 'vs/editor/common/modes';
 import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
+import {LineStream} from 'vs/editor/common/modes/lineStream';
 
 // --------- utils
 
@@ -38,7 +39,7 @@ suite('Editor Model - Model Modes 1', () => {
 		public equals(other: modes.IState): boolean {
 			return this === other;
 		}
-		public tokenize(stream:modes.IStream): ITokenizationResult {
+		public tokenize(stream:LineStream): ITokenizationResult {
 			calledState.calledFor.push(stream.next());
 			stream.advanceToEOS();
 			return { type: '' };
@@ -188,7 +189,7 @@ suite('Editor Model - Model Modes 2', () => {
 			return (other instanceof ModelState2) && (this.prevLineContent === (<ModelState2>other).prevLineContent);
 		}
 
-		public tokenize(stream:modes.IStream):ITokenizationResult {
+		public tokenize(stream:LineStream):ITokenizationResult {
 			var line= '';
 			while (!stream.eos()) {
 				line+= stream.next();
@@ -325,7 +326,7 @@ suite('Editor Model - Token Iterator', () => {
 			return true;
 		}
 
-		public tokenize(stream:modes.IStream):ITokenizationResult {
+		public tokenize(stream:LineStream):ITokenizationResult {
 			var ndash = this.n, value = '';
 			while(!stream.eos() && ndash > 0) {
 				value += stream.next();

src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts

@@ -5,11 +5,12 @@
 'use strict';
 
 import * as assert from 'assert';
-import {IStream, TokenizationRegistry} from 'vs/editor/common/modes';
+import {TokenizationRegistry} from 'vs/editor/common/modes';
 import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
 import {tokenizeToHtmlContent} from 'vs/editor/common/modes/textToHtmlTokenizer';
 import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
+import {LineStream} from 'vs/editor/common/modes/lineStream';
 
 suite('Editor Modes - textToHtmlTokenizer', () => {
 	test('TextToHtmlTokenizer', () => {
@@ -67,7 +68,7 @@ class State extends AbstractState {
 		return new State(this.getModeId());
 	}
 
-	public tokenize(stream:IStream):ITokenizationResult {
+	public tokenize(stream:LineStream):ITokenizationResult {
 		return { type: stream.next() === '.' ? '' : 'text' };
 	}
 }

src/vs/editor/test/common/modes/tokenization.test.ts

@@ -13,6 +13,7 @@ import {createMockLineContext} from 'vs/editor/test/common/modesTestUtils';
 import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
 import {Token} from 'vs/editor/common/core/token';
+import {LineStream} from 'vs/editor/common/modes/lineStream';
 
 export interface IModeSwitchingDescriptor {
 	[character:string]:{
@@ -36,7 +37,7 @@ export class StateMemorizingLastWord extends AbstractState {
 		return new StateMemorizingLastWord(this.getModeId(), this.descriptor, this.lastWord);
 	}
 
-	public tokenize(stream:modes.IStream):ITokenizationResult {
+	public tokenize(stream:LineStream):ITokenizationResult {
 		stream.setTokenRules('[]{}()==--', '\t \u00a0');
 		if (stream.skipWhitespace() !== '') {
 			return {
@@ -167,7 +168,7 @@ suite('Editor Modes - Tokenization', () => {
 				return new State(this.getModeId());
 			}
 
-			public tokenize(stream:modes.IStream):ITokenizationResult {
+			public tokenize(stream:LineStream):ITokenizationResult {
 				return { type: stream.next() === '.' ? '' : 'text' };
 			}
 		}