Fixes #3048 [css] Uncaught RangeError: Maximum call stack size exceeded

src/vs/languages/css/common/parser/cssScanner.ts

@@ -46,6 +46,7 @@ export enum TokenType {
 
 	EscapedJavaScript,
 	BadEscapedJavaScript,
+	Comment,
 	SingleLineComment,
 	EOF,
 	CustomToken
@@ -115,7 +116,7 @@ export class MultiLineStream {
 	}
 
 	public advanceIfChars(ch:number[]):boolean {
-		var i:number;
+		let i:number;
 		if (this.position + ch.length > this.source.length) {
 			return false;
 		}
@@ -129,7 +130,7 @@ export class MultiLineStream {
 	}
 
 	public advanceWhileChar(condition:(ch:number)=>boolean):number {
-		var posNow = this.position;
+		let posNow = this.position;
 		while (this.position < this.len && condition(this.source.charCodeAt(this.position))) {
 			this.position++;
 		}
@@ -137,66 +138,66 @@ export class MultiLineStream {
 	}
 }
 
-var _a = 'a'.charCodeAt(0);
-var _e = 'e'.charCodeAt(0);
-var _f = 'f'.charCodeAt(0);
-var _i = 'i'.charCodeAt(0);
-var _l = 'l'.charCodeAt(0);
-var _p = 'p'.charCodeAt(0);
-var _r = 'r'.charCodeAt(0);
-var _u = 'u'.charCodeAt(0);
-var _x = 'x'.charCodeAt(0);
-var _z = 'z'.charCodeAt(0);
-var _A = 'A'.charCodeAt(0);
-var _E = 'E'.charCodeAt(0);
-var _F = 'F'.charCodeAt(0);
-var _I = 'I'.charCodeAt(0);
-var _L = 'L'.charCodeAt(0);
-var _P = 'P'.charCodeAt(0);
-var _R = 'R'.charCodeAt(0);
-var _U = 'U'.charCodeAt(0);
-var _X = 'X'.charCodeAt(0);
-var _Z = 'Z'.charCodeAt(0);
-var _0 = '0'.charCodeAt(0);
-var _9 = '9'.charCodeAt(0);
-var _TLD = '~'.charCodeAt(0);
-var _HAT = '^'.charCodeAt(0);
-var _EQS = '='.charCodeAt(0);
-var _PIP = '|'.charCodeAt(0);
-var _MIN = '-'.charCodeAt(0);
-var _USC = '_'.charCodeAt(0);
-var _PRC = '%'.charCodeAt(0);
-var _MUL = '*'.charCodeAt(0);
-var _LPA = '('.charCodeAt(0);
-var _RPA = ')'.charCodeAt(0);
-var _LAN = '<'.charCodeAt(0);
-var _RAN = '>'.charCodeAt(0);
-var _ATS = '@'.charCodeAt(0);
-var _HSH = '#'.charCodeAt(0);
-var _DLR = '$'.charCodeAt(0);
-var _BSL = '\\'.charCodeAt(0);
-var _FSL = '/'.charCodeAt(0);
-var _NWL = '\n'.charCodeAt(0);
-var _CAR = '\r'.charCodeAt(0);
-var _LFD = '\f'.charCodeAt(0);
-var _DQO = '"'.charCodeAt(0);
-var _SQO = '\''.charCodeAt(0);
-var _WSP = ' '.charCodeAt(0);
-var _TAB = '\t'.charCodeAt(0);
-var _SEM = ';'.charCodeAt(0);
-var _COL = ':'.charCodeAt(0);
-var _CUL = '{'.charCodeAt(0);
-var _CUR = '}'.charCodeAt(0);
-var _BRL = '['.charCodeAt(0);
-var _BRR = ']'.charCodeAt(0);
-var _CMA = ','.charCodeAt(0);
-var _DOT = '.'.charCodeAt(0);
-var _BNG = '!'.charCodeAt(0);
-
-var _url = [_u, _U, _r, _R, _l, _L, _LPA, _LPA];
-var _url_prefix = [_u, _U, _r, _R, _l, _L, _MIN, _MIN, _p, _P, _r, _R, _e, _E, _f, _F, _i, _I, _x, _X, _LPA, _LPA];
-
-var staticTokenTable:{[code:number]:TokenType;} = {};
+const _a = 'a'.charCodeAt(0);
+const _e = 'e'.charCodeAt(0);
+const _f = 'f'.charCodeAt(0);
+const _i = 'i'.charCodeAt(0);
+const _l = 'l'.charCodeAt(0);
+const _p = 'p'.charCodeAt(0);
+const _r = 'r'.charCodeAt(0);
+const _u = 'u'.charCodeAt(0);
+const _x = 'x'.charCodeAt(0);
+const _z = 'z'.charCodeAt(0);
+const _A = 'A'.charCodeAt(0);
+const _E = 'E'.charCodeAt(0);
+const _F = 'F'.charCodeAt(0);
+const _I = 'I'.charCodeAt(0);
+const _L = 'L'.charCodeAt(0);
+const _P = 'P'.charCodeAt(0);
+const _R = 'R'.charCodeAt(0);
+const _U = 'U'.charCodeAt(0);
+const _X = 'X'.charCodeAt(0);
+const _Z = 'Z'.charCodeAt(0);
+const _0 = '0'.charCodeAt(0);
+const _9 = '9'.charCodeAt(0);
+const _TLD = '~'.charCodeAt(0);
+const _HAT = '^'.charCodeAt(0);
+const _EQS = '='.charCodeAt(0);
+const _PIP = '|'.charCodeAt(0);
+const _MIN = '-'.charCodeAt(0);
+const _USC = '_'.charCodeAt(0);
+const _PRC = '%'.charCodeAt(0);
+const _MUL = '*'.charCodeAt(0);
+const _LPA = '('.charCodeAt(0);
+const _RPA = ')'.charCodeAt(0);
+const _LAN = '<'.charCodeAt(0);
+const _RAN = '>'.charCodeAt(0);
+const _ATS = '@'.charCodeAt(0);
+const _HSH = '#'.charCodeAt(0);
+const _DLR = '$'.charCodeAt(0);
+const _BSL = '\\'.charCodeAt(0);
+const _FSL = '/'.charCodeAt(0);
+const _NWL = '\n'.charCodeAt(0);
+const _CAR = '\r'.charCodeAt(0);
+const _LFD = '\f'.charCodeAt(0);
+const _DQO = '"'.charCodeAt(0);
+const _SQO = '\''.charCodeAt(0);
+const _WSP = ' '.charCodeAt(0);
+const _TAB = '\t'.charCodeAt(0);
+const _SEM = ';'.charCodeAt(0);
+const _COL = ':'.charCodeAt(0);
+const _CUL = '{'.charCodeAt(0);
+const _CUR = '}'.charCodeAt(0);
+const _BRL = '['.charCodeAt(0);
+const _BRR = ']'.charCodeAt(0);
+const _CMA = ','.charCodeAt(0);
+const _DOT = '.'.charCodeAt(0);
+const _BNG = '!'.charCodeAt(0);
+
+const _url = [_u, _U, _r, _R, _l, _L, _LPA, _LPA];
+const _url_prefix = [_u, _U, _r, _R, _l, _L, _MIN, _MIN, _p, _P, _r, _R, _e, _E, _f, _F, _i, _I, _x, _X, _LPA, _LPA];
+
+const staticTokenTable:{[code:number]:TokenType;} = {};
 staticTokenTable[_SEM] = TokenType.SemiColon;
 staticTokenTable[_COL] = TokenType.Colon;
 staticTokenTable[_CUL] = TokenType.CurlyL;
@@ -207,7 +208,7 @@ staticTokenTable[_LPA] = TokenType.ParenthesisL;
 staticTokenTable[_RPA] = TokenType.ParenthesisR;
 staticTokenTable[_CMA] = TokenType.Comma;
 
-var staticUnitTable:{[code:number]:TokenType;} = {};
+const staticUnitTable:{[code:number]:TokenType;} = {};
 staticUnitTable['em'] = TokenType.EMS;
 staticUnitTable['ex'] = TokenType.EXS;
 staticUnitTable['px'] = TokenType.Length;
@@ -231,16 +232,19 @@ export class Scanner {
 
 	public stream: MultiLineStream;
 	public ignoreComment = true;
+	public ignoreWhitespace = true;
 
 	public setSource(input: string): void {
 		this.stream = new MultiLineStream(input);
 	}
 
-	public finishToken(token: IToken, type: TokenType, text?: string): IToken {
-		token.len = this.stream.pos() - token.offset;
-		token.type = type;
-		token.text = text || this.stream.substring(token.offset);
-		return token;
+	public finishToken(offset: number, type: TokenType, text?: string): IToken {
+		return {
+			offset: offset,
+			len: this.stream.pos() - offset,
+			type: type,
+			text: text || this.stream.substring(offset)
+		};
 	}
 
 	public substring(offset:number, len:number):string {
@@ -255,71 +259,51 @@ export class Scanner {
 		this.stream.goBackTo(pos);
 	}
 
-	public scan(ignoreWhitespace:boolean=true): IToken {
-
-		var result:IToken = {
-			type: undefined,
-			text: undefined,
-			offset: this.stream.pos(),
-			len: 0
-		};
-
-		// Whitespace - if asked for
-		if (this._whitespace()) {
-			if (!ignoreWhitespace) {
-				return this.finishToken(result, TokenType.Whitespace);
-			} else {
-				return this.scan(ignoreWhitespace);
+	public scan(): IToken {
+		// processes all whitespaces and comments
+		let triviaToken = this.trivia();
+		if (triviaToken !== null) {
+			return triviaToken;
 		}
-		}
-
-		let tokenType: TokenType = void 0;
 
-		// Comment - CSS
-		if (this._comment()) {
-			if (!this.ignoreComment) {
-				return this.finishToken(result, tokenType);
-			} else {
-				return this.scan(ignoreWhitespace);
-			}
-		}
+		let offset = this.stream.pos();
 
 		// End of file/input
 		if (this.stream.eos()) {
-			return this.finishToken(result, TokenType.EOF);
+			return this.finishToken(offset, TokenType.EOF);
 		}
 
 		// CDO <!--
 		if (this.stream.advanceIfChars([_LAN, _BNG, _MIN, _MIN])) {
-			return this.finishToken(result, TokenType.CDO);
+			return this.finishToken(offset, TokenType.CDO);
 		}
 
 		// CDC -->
 		if (this.stream.advanceIfChars([_MIN, _MIN, _RAN])) {
-			return this.finishToken(result, TokenType.CDC);
+			return this.finishToken(offset, TokenType.CDC);
 		}
 
 		// URL
-		tokenType = this._url();
+		let tokenType = this._url();
 		if (tokenType !== null) {
-			return this.finishToken(result, tokenType);
+			return this.finishToken(offset, tokenType);
 		}
-		var content: string[] = [];
+		let content: string[] = [];
 		if (this.ident(content)) {
-			return this.finishToken(result, TokenType.Ident, content.join(''));
+			return this.finishToken(offset, TokenType.Ident, content.join(''));
 		}
 
 		// at-keyword
 		if (this.stream.advanceIfChar(_ATS)) {
 			content = [ '@' ];
 			if (this.ident(content)) {
-				var keywordText = content.join('');
+				let keywordText = content.join('');
 				if (keywordText === '@charset') {
-					return this.finishToken(result, TokenType.Charset, keywordText);
+					return this.finishToken(offset, TokenType.Charset, keywordText);
 				}
-				return this.finishToken(result, TokenType.AtKeyword, keywordText);
+				return this.finishToken(offset, TokenType.AtKeyword, keywordText);
 			} else {
-				return this.finishToken(result, TokenType.Delim);
+				return this.finishToken(offset, TokenType.Delim);
 			}
 		}
 
@@ -327,93 +311,93 @@ export class Scanner {
 		if (this.stream.advanceIfChar(_HSH)) {
 			content = [ '#' ];
 			if (this._name(content)) {
-				return this.finishToken(result, TokenType.Hash, content.join(''));
+				return this.finishToken(offset, TokenType.Hash, content.join(''));
 			} else {
-				return this.finishToken(result, TokenType.Delim);
+				return this.finishToken(offset, TokenType.Delim);
 			}
 		}
 
 		// Important
 		if (this.stream.advanceIfChar(_BNG)) {
-			return this.finishToken(result, TokenType.Exclamation);
+			return this.finishToken(offset, TokenType.Exclamation);
 		}
 
 		// Numbers
 		if (this._number()) {
 
-			var pos = this.stream.pos();
-			content = [ this.stream.substring(result.offset, pos) ];
+			let pos = this.stream.pos();
+			content = [ this.stream.substring(offset, pos) ];
 			if (this.stream.advanceIfChar(_PRC)) {
 				// Percentage 43%
-				return this.finishToken(result, TokenType.Percentage);
+				return this.finishToken(offset, TokenType.Percentage);
 			} else if (this.ident(content)) {
 				let dim = this.stream.substring(pos).toLowerCase();
 				tokenType = <TokenType>staticUnitTable[dim];
 				if (typeof tokenType !== 'undefined') {
 					// Known dimension 43px
-					return this.finishToken(result, tokenType, content.join(''));
+					return this.finishToken(offset, tokenType, content.join(''));
 				} else {
 					// Unknown dimension 43ft
-					return this.finishToken(result, TokenType.Dimension, content.join(''));
+					return this.finishToken(offset, TokenType.Dimension, content.join(''));
 				}
 			}
 
-			return this.finishToken(result, TokenType.Num);
+			return this.finishToken(offset, TokenType.Num);
 		}
 
 		// String, BadString
 		content = [];
 		tokenType = this._string(content);
 		if (tokenType !== null) {
-			return this.finishToken(result, tokenType, content.join(''));
+			return this.finishToken(offset, tokenType, content.join(''));
 		}
 
 		// single character tokens
 		tokenType = <TokenType>staticTokenTable[this.stream.peekChar()];
 		if (typeof tokenType !== 'undefined') {
 			this.stream.advance(1);
-			return this.finishToken(result, tokenType);
+			return this.finishToken(offset, tokenType);
 		}
 
 		// includes ~=
 		if (this.stream.peekChar(0) === _TLD && this.stream.peekChar(1) === _EQS) {
 			this.stream.advance(2);
-			return this.finishToken(result, TokenType.Includes);
+			return this.finishToken(offset, TokenType.Includes);
 		}
 
 		// DashMatch |=
 		if (this.stream.peekChar(0) === _PIP && this.stream.peekChar(1) === _EQS) {
 			this.stream.advance(2);
-			return this.finishToken(result, TokenType.Dashmatch);
+			return this.finishToken(offset, TokenType.Dashmatch);
 		}
 
 		// Substring operator *=
 		if (this.stream.peekChar(0) === _MUL && this.stream.peekChar(1) === _EQS) {
 			this.stream.advance(2);
-			return this.finishToken(result, TokenType.SubstringOperator);
+			return this.finishToken(offset, TokenType.SubstringOperator);
 		}
 
 		// Substring operator ^=
 		if (this.stream.peekChar(0) === _HAT && this.stream.peekChar(1) === _EQS) {
 			this.stream.advance(2);
-			return this.finishToken(result, TokenType.PrefixOperator);
+			return this.finishToken(offset, TokenType.PrefixOperator);
 		}
 
 		// Substring operator $=
 		if (this.stream.peekChar(0) === _DLR && this.stream.peekChar(1) === _EQS) {
 			this.stream.advance(2);
-			return this.finishToken(result, TokenType.SuffixOperator);
+			return this.finishToken(offset, TokenType.SuffixOperator);
 		}
 
 		// Delim
 		this.stream.nextChar();
-		return this.finishToken(result, TokenType.Delim);
+		return this.finishToken(offset, TokenType.Delim);
 	}
 
 	private _matchWordAnyCase(characters:number[]): boolean {
-		var index = 0;
+		let index = 0;
 		this.stream.advanceWhileChar((ch:number) => {
-			var result = characters[index] === ch || characters[index+1] === ch;
+			let result = characters[index] === ch || characters[index+1] === ch;
 			if (result) {
 				index += 2;
 			}
@@ -427,9 +411,26 @@ export class Scanner {
 		}
 	}
 
-	private _comment():boolean {
+	protected trivia(): IToken {
+		while (true) {
+			let offset = this.stream.pos();
+			if (this._whitespace()) {
+				if (!this.ignoreWhitespace) {
+					return this.finishToken(offset, TokenType.Whitespace);
+				}
+			} else if (this.comment()) {
+				if (!this.ignoreComment) {
+					return this.finishToken(offset, TokenType.Comment);
+				}
+			} else {
+				return null;
+			}
+		}
+	}
+
+	protected comment():boolean {
 		if (this.stream.advanceIfChars([_FSL, _MUL])) {
-			var success = false, hot = false;
+			let success = false, hot = false;
 			this.stream.advanceWhileChar((ch) => {
 				if (hot && ch === _FSL) {
 					success = true;
@@ -448,7 +449,7 @@ export class Scanner {
 	}
 
 	private _number():boolean {
-		var npeek = 0, ch:number;
+		let npeek = 0, ch:number;
 		if (this.stream.peekChar() === _DOT) {
 			npeek = 1;
 		}
@@ -464,7 +465,7 @@ export class Scanner {
 	}
 
 	private _newline(result: string[]):boolean {
-		var ch = this.stream.peekChar();
+		let ch = this.stream.peekChar();
 		switch (ch) {
 			case _CAR:
 			case _LFD:
@@ -482,11 +483,11 @@ export class Scanner {
 	}
 
 	private _escape(result: string[], includeNewLines?:boolean):boolean {
-		var ch = this.stream.peekChar();
+		let ch = this.stream.peekChar();
 		if (ch === _BSL) {
 			this.stream.advance(1);
 			ch = this.stream.peekChar();
-			var hexNumCount = 0;
+			let hexNumCount = 0;
 			while (hexNumCount < 6 && (ch >= _0 && ch <= _9 || ch >= _a && ch <= _f || ch >= _A && ch <= _F)) {
 				this.stream.advance(1);
 				ch = this.stream.peekChar();
@@ -494,7 +495,7 @@ export class Scanner {
 			}
 			if (hexNumCount > 0) {
 				try {
-					var hexVal= parseInt(this.stream.substring(this.stream.pos() - hexNumCount), 16);
+					let hexVal= parseInt(this.stream.substring(this.stream.pos() - hexNumCount), 16);
 					if (hexVal) {
 						result.push(String.fromCharCode(hexVal));
 					}
@@ -523,7 +524,7 @@ export class Scanner {
 
 	private _stringChar(closeQuote: number, result: string[])  {
 		// not closeQuote, not backslash, not newline
-		var ch = this.stream.peekChar();
+		let ch = this.stream.peekChar();
 		if (ch !== 0 && ch !== closeQuote && ch !== _BSL && ch !== _CAR && ch !== _LFD && ch !== _NWL) {
 			this.stream.advance(1);
 			result.push(String.fromCharCode(ch));
@@ -534,7 +535,7 @@ export class Scanner {
 
 	private _string(result: string[]):TokenType {
 		if (this.stream.peekChar() === _SQO || this.stream.peekChar() === _DQO) {
-			var closeQuote = this.stream.nextChar();
+			let closeQuote = this.stream.nextChar();
 			result.push(String.fromCharCode(closeQuote));
 
 			while (this._stringChar(closeQuote, result) || this._escape(result, true)) {
@@ -555,7 +556,7 @@ export class Scanner {
 	private _url():TokenType {
 		if (this._matchWordAnyCase(_url) || this._matchWordAnyCase(_url_prefix)) {
 			this._whitespace();
-			var tokenType = TokenType.URI, stringType = this._string([]);
+			let tokenType = TokenType.URI, stringType = this._string([]);
 			if (stringType === TokenType.BadString) {
 				tokenType = TokenType.BadUri;
 
@@ -577,14 +578,14 @@ export class Scanner {
 	}
 
 	private _whitespace():boolean {
-		var n = this.stream.advanceWhileChar((ch) => {
+		let n = this.stream.advanceWhileChar((ch) => {
 			return ch === _WSP || ch === _TAB || ch === _NWL || ch === _LFD || ch === _CAR;
 		});
 		return n > 0;
 	}
 
 	private _name(result:string[]):boolean {
-		var matched = false;
+		let matched = false;
 		while (this._identChar(result) || this._escape(result)) {
 			matched = true;
 		}
@@ -592,10 +593,10 @@ export class Scanner {
 	}
 
 	protected ident(result:string[]):boolean {
-		var pos = this.stream.pos();
-		var hasMinus = this._minus(result);
+		let pos = this.stream.pos();
+		let hasMinus = this._minus(result);
 		if (hasMinus && this._minus(result) /* -- */) {
-			var hasContent = false;
+			let hasContent = false;
 			while (this._identChar(result) || this._escape(result)) {
 				hasContent = true;
 			}
@@ -613,7 +614,7 @@ export class Scanner {
 	}
 
 	private _identFirstChar(result:string[]):boolean {
-		var ch = this.stream.peekChar();
+		let ch = this.stream.peekChar();
 		if (ch === _USC || // _
 			ch >= _a && ch <= _z || // a-z
 			ch >= _A && ch <= _Z || // A-Z
@@ -627,7 +628,7 @@ export class Scanner {
 
 
 	private _minus(result:string[]):boolean {
-		var ch = this.stream.peekChar();
+		let ch = this.stream.peekChar();
 		if (ch === _MIN) {
 			this.stream.advance(1);
 			result.push(String.fromCharCode(ch));
@@ -637,7 +638,7 @@ export class Scanner {
 	}
 
 	private _identChar(result:string[]):boolean {
-		var ch = this.stream.peekChar();
+		let ch = this.stream.peekChar();
 		if (ch === _USC || // _
 			ch === _MIN || // -
 			ch >= _a && ch <= _z || // a-z

src/vs/languages/css/test/common/scanner.test.ts

@@ -10,9 +10,9 @@ import Scanner = require('vs/languages/css/common/parser/cssScanner');
 
 suite('CSS - Scanner', () => {
 
-	function assertSingleToken(scan: Scanner.Scanner, source: string, len: number, offset: number, text: string, type: Scanner.TokenType, ignoreWhitespace?:boolean):void {
+	function assertSingleToken(scan: Scanner.Scanner, source: string, len: number, offset: number, text: string, type: Scanner.TokenType):void {
 		scan.setSource(source);
-		var token = scan.scan(ignoreWhitespace);
+		var token = scan.scan();
 		assert.equal(token.len, len);
 		assert.equal(token.offset, offset);
 		assert.equal(token.text, text);
@@ -22,7 +22,17 @@ suite('CSS - Scanner', () => {
 	test('Test Whitespace', function() {
 		var scanner = new Scanner.Scanner();
 		assertSingleToken(scanner, ' @', 1, 1, '@', Scanner.TokenType.Delim);
-		assertSingleToken(scanner, ' @', 1, 0, ' ', Scanner.TokenType.Whitespace, false);
+		assertSingleToken(scanner, ' /* comment*/ \n/*comment*/@', 1, 26, '@', Scanner.TokenType.Delim);
+
+		scanner = new Scanner.Scanner();
+		scanner.ignoreWhitespace = false;
+		assertSingleToken(scanner, ' @', 1, 0, ' ', Scanner.TokenType.Whitespace);
+		assertSingleToken(scanner, '/*comment*/ @', 1, 11, ' ', Scanner.TokenType.Whitespace);
+
+		scanner = new Scanner.Scanner();
+		scanner.ignoreComment = false;
+		assertSingleToken(scanner, ' /*comment*/@', 11, 1, '/*comment*/', Scanner.TokenType.Comment);
+		assertSingleToken(scanner, '/*comment*/ @', 11, 0, '/*comment*/', Scanner.TokenType.Comment);
 	});
 
 	test('Test Token Ident', function() {

src/vs/languages/less/common/parser/lessScanner.ts

@@ -6,50 +6,44 @@
 
 import scanner = require('vs/languages/css/common/parser/cssScanner');
 
-var _FSL = '/'.charCodeAt(0);
-var _NWL = '\n'.charCodeAt(0);
-var _CAR = '\r'.charCodeAt(0);
-var _LFD = '\f'.charCodeAt(0);
-var _TIC = '`'.charCodeAt(0);
-var _DOT = '.'.charCodeAt(0);
+const _FSL = '/'.charCodeAt(0);
+const _NWL = '\n'.charCodeAt(0);
+const _CAR = '\r'.charCodeAt(0);
+const _LFD = '\f'.charCodeAt(0);
+const _TIC = '`'.charCodeAt(0);
+const _DOT = '.'.charCodeAt(0);
 
-var customTokenValue = scanner.TokenType.CustomToken;
-export var Ellipsis: scanner.TokenType = customTokenValue++;
+let customTokenValue = scanner.TokenType.CustomToken;
+export const Ellipsis: scanner.TokenType = customTokenValue++;
 
 export class LessScanner extends scanner.Scanner {
 
-	public scan(ignoreWhitespace:boolean=true): scanner.IToken {
+	public scan(): scanner.IToken {
 
-		var result:scanner.IToken = {
-			type: undefined,
-			text: undefined,
-			offset: this.stream.pos(),
-			len: 0
-		};
-
-		// SingleLine Comments
-		if (this.lessComment()) {
-			if (!this.ignoreComment) {
-				return this.finishToken(result, scanner.TokenType.SingleLineComment);
-			} else {
-				return this.scan(ignoreWhitespace);
-			}
+		let triviaToken = this.trivia();
+		if (triviaToken !== null) {
+			return triviaToken;
 		}
 
-		// LESS: escaped JavaScript code `var a = "dddd"`
-		var tokenType = this.escapedJavaScript();
-		if(tokenType !== null) {
-			return this.finishToken(result, tokenType);
+		let offset = this.stream.pos();
+
+		// LESS: escaped JavaScript code `let a = "dddd"`
+		let tokenType = this.escapedJavaScript();
+		if (tokenType !== null) {
+			return this.finishToken(offset, tokenType);
 		}
 
-		if(this.stream.advanceIfChars([_DOT, _DOT, _DOT])) {
-			return this.finishToken(result, Ellipsis);
+		if (this.stream.advanceIfChars([_DOT, _DOT, _DOT])) {
+			return this.finishToken(offset, Ellipsis);
 		}
 
-		return super.scan(ignoreWhitespace);
+		return super.scan();
 	}
 
-	private lessComment():boolean {
+	protected comment():boolean {
+		if (super.comment()) {
+			return true;
+		}
 		if (this.stream.advanceIfChars([_FSL, _FSL])) {
 			this.stream.advanceWhileChar((ch:number) => {
 				switch(ch) {
@@ -68,8 +62,8 @@ export class LessScanner extends scanner.Scanner {
 	}
 
 	private escapedJavaScript():scanner.TokenType {
-		var ch = this.stream.peekChar();
-		if(ch === _TIC) {
+		let ch = this.stream.peekChar();
+		if (ch === _TIC) {
 			this.stream.advance(1);
 			this.stream.advanceWhileChar((ch) => { return ch !== _TIC; });
 			return this.stream.advanceIfChar(_TIC) ? scanner.TokenType.EscapedJavaScript : scanner.TokenType.BadEscapedJavaScript;

src/vs/languages/less/test/common/scanner.test.ts

@@ -9,10 +9,10 @@ import assert = require('assert');
 import scanner = require('vs/languages/css/common/parser/cssScanner');
 import lessScanner = require('vs/languages/less/common/parser/lessScanner');
 
-function assertSingleToken(source: string, len: number, offset: number, text: string, type: scanner.TokenType, ignoreWhitespace?:boolean):void {
+function assertSingleToken(source: string, len: number, offset: number, text: string, type: scanner.TokenType):void {
 	var scan = new lessScanner.LessScanner();
 	scan.setSource(source);
-	var token = scan.scan(ignoreWhitespace);
+	var token = scan.scan();
 	assert.equal(token.len, len);
 	assert.equal(token.offset, offset);
 	assert.equal(token.text, text);
@@ -35,5 +35,6 @@ suite('LESS - Scanner', () => {
 		assertSingleToken('//this is a comment test', 0, 24, '', scanner.TokenType.EOF);
 		assertSingleToken('// this is a comment test', 0, 25, '', scanner.TokenType.EOF);
 		assertSingleToken('// this is a\na', 1, 13, 'a', scanner.TokenType.Ident);
+		assertSingleToken('// this is a\n// more\n   \n/* comment */a', 1, 38, 'a', scanner.TokenType.Ident);
 	});
 });

src/vs/languages/sass/common/parser/sassScanner.ts

@@ -6,101 +6,96 @@
 
 import scanner = require('vs/languages/css/common/parser/cssScanner');
 
-var _FSL = '/'.charCodeAt(0);
-var _NWL = '\n'.charCodeAt(0);
-var _CAR = '\r'.charCodeAt(0);
-var _LFD = '\f'.charCodeAt(0);
-
-var _DLR = '$'.charCodeAt(0);
-var _HSH = '#'.charCodeAt(0);
-var _CUL = '{'.charCodeAt(0);
-var _EQS = '='.charCodeAt(0);
-var _BNG = '!'.charCodeAt(0);
-var _LAN = '<'.charCodeAt(0);
-var _RAN = '>'.charCodeAt(0);
-var _DOT = '.'.charCodeAt(0);
-
-var customTokenValue = scanner.TokenType.CustomToken;
-
-export var VariableName = customTokenValue++;
-export var InterpolationFunction: scanner.TokenType = customTokenValue++;
-export var Default: scanner.TokenType = customTokenValue++;
-export var EqualsOperator: scanner.TokenType = customTokenValue++;
-export var NotEqualsOperator: scanner.TokenType = customTokenValue++;
-export var GreaterEqualsOperator: scanner.TokenType = customTokenValue++;
-export var SmallerEqualsOperator: scanner.TokenType = customTokenValue++;
-export var Ellipsis: scanner.TokenType = customTokenValue++;
+const _FSL = '/'.charCodeAt(0);
+const _NWL = '\n'.charCodeAt(0);
+const _CAR = '\r'.charCodeAt(0);
+const _LFD = '\f'.charCodeAt(0);
+
+const _DLR = '$'.charCodeAt(0);
+const _HSH = '#'.charCodeAt(0);
+const _CUL = '{'.charCodeAt(0);
+const _EQS = '='.charCodeAt(0);
+const _BNG = '!'.charCodeAt(0);
+const _LAN = '<'.charCodeAt(0);
+const _RAN = '>'.charCodeAt(0);
+const _DOT = '.'.charCodeAt(0);
+
+let customTokenValue = scanner.TokenType.CustomToken;
+
+export const VariableName = customTokenValue++;
+export const InterpolationFunction: scanner.TokenType = customTokenValue++;
+export const Default: scanner.TokenType = customTokenValue++;
+export const EqualsOperator: scanner.TokenType = customTokenValue++;
+export const NotEqualsOperator: scanner.TokenType = customTokenValue++;
+export const GreaterEqualsOperator: scanner.TokenType = customTokenValue++;
+export const SmallerEqualsOperator: scanner.TokenType = customTokenValue++;
+export const Ellipsis: scanner.TokenType = customTokenValue++;
 
 export class SassScanner extends scanner.Scanner {
 
-	public scan(ignoreWhitespace:boolean=true): scanner.IToken {
+	public scan(): scanner.IToken {
 
-		var result:scanner.IToken = {
-			type: undefined,
-			text: undefined,
-			offset: this.stream.pos(),
-			len: 0
-		};
-
-		// SingleLine Comments
-		if (this.sassComment()) {
-			if (!this.ignoreComment) {
-				return this.finishToken(result, scanner.TokenType.SingleLineComment);
-			} else {
-				return this.scan(ignoreWhitespace);
-			}
+		// processes all whitespaces and comments
+		const triviaToken = this.trivia();
+		if (triviaToken !== null) {
+			return triviaToken;
 		}
 
+		const offset = this.stream.pos();
+
 		// sass variable
 		if (this.stream.advanceIfChar(_DLR)) {
-			var content = [ '$' ];
+			const content = [ '$' ];
 			if (this.ident(content)) {
-				return this.finishToken(result, VariableName, content.join(''));
+				return this.finishToken(offset, VariableName, content.join(''));
 			} else {
-				this.stream.goBackTo(result.offset);
+				this.stream.goBackTo(offset);
 			}
 		}
 
 		// Sass: interpolation function #{..})
 		if (this.stream.advanceIfChars([_HSH, _CUL])) {
-			return this.finishToken(result, InterpolationFunction);
+			return this.finishToken(offset, InterpolationFunction);
 		}
 
 		// operator ==
 		if (this.stream.advanceIfChars([_EQS, _EQS])) {
-			return this.finishToken(result, EqualsOperator);
+			return this.finishToken(offset, EqualsOperator);
 		}
 
 		// operator !=
 		if (this.stream.advanceIfChars([_BNG, _EQS])) {
-			return this.finishToken(result, NotEqualsOperator);
+			return this.finishToken(offset, NotEqualsOperator);
 		}
 
 		// operators <, <=
 		if (this.stream.advanceIfChar(_LAN)) {
 			if (this.stream.advanceIfChar(_EQS)) {
-				return this.finishToken(result, SmallerEqualsOperator);
+				return this.finishToken(offset, SmallerEqualsOperator);
 			}
-			return this.finishToken(result, scanner.TokenType.Delim);
+			return this.finishToken(offset, scanner.TokenType.Delim);
 		}
 
 		// ooperators >, >=
 		if (this.stream.advanceIfChar(_RAN)) {
 			if (this.stream.advanceIfChar(_EQS)) {
-				return this.finishToken(result, GreaterEqualsOperator);
+				return this.finishToken(offset, GreaterEqualsOperator);
 			}
-			return this.finishToken(result, scanner.TokenType.Delim);
+			return this.finishToken(offset, scanner.TokenType.Delim);
 		}
 
 		// ellipis
 		if (this.stream.advanceIfChars([_DOT, _DOT, _DOT])) {
-			return this.finishToken(result, Ellipsis);
+			return this.finishToken(offset, Ellipsis);
 		}
 
-		return super.scan(ignoreWhitespace);
+		return super.scan();
 	}
 
-	private sassComment():boolean {
+	protected comment():boolean {
+		if (super.comment()) {
+			return true;
+		}
 		if (this.stream.advanceIfChars([_FSL, _FSL])) {
 			this.stream.advanceWhileChar((ch:number) => {
 				switch(ch) {