From 1139f8bbed0de4d528ea247f3e592ff78592edb3 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Mon, 22 Feb 2016 12:41:54 +0100 Subject: [PATCH] Fixes #3048 [css] Uncaught RangeError: Maximum call stack size exceeded --- .../languages/css/common/parser/cssScanner.ts | 285 +++++++++--------- .../languages/css/test/common/scanner.test.ts | 16 +- .../less/common/parser/lessScanner.ts | 60 ++-- .../less/test/common/scanner.test.ts | 5 +- .../sass/common/parser/sassScanner.ts | 99 +++--- 5 files changed, 233 insertions(+), 232 deletions(-) diff --git a/src/vs/languages/css/common/parser/cssScanner.ts b/src/vs/languages/css/common/parser/cssScanner.ts index 7af38068e7d..c2ee8abeda8 100644 --- a/src/vs/languages/css/common/parser/cssScanner.ts +++ b/src/vs/languages/css/common/parser/cssScanner.ts @@ -46,6 +46,7 @@ export enum TokenType { EscapedJavaScript, BadEscapedJavaScript, + Comment, SingleLineComment, EOF, CustomToken @@ -115,7 +116,7 @@ export class MultiLineStream { } public advanceIfChars(ch:number[]):boolean { - var i:number; + let i:number; if (this.position + ch.length > this.source.length) { return false; } @@ -129,7 +130,7 @@ export class MultiLineStream { } public advanceWhileChar(condition:(ch:number)=>boolean):number { - var posNow = this.position; + let posNow = this.position; while (this.position < this.len && condition(this.source.charCodeAt(this.position))) { this.position++; } @@ -137,66 +138,66 @@ export class MultiLineStream { } } -var _a = 'a'.charCodeAt(0); -var _e = 'e'.charCodeAt(0); -var _f = 'f'.charCodeAt(0); -var _i = 'i'.charCodeAt(0); -var _l = 'l'.charCodeAt(0); -var _p = 'p'.charCodeAt(0); -var _r = 'r'.charCodeAt(0); -var _u = 'u'.charCodeAt(0); -var _x = 'x'.charCodeAt(0); -var _z = 'z'.charCodeAt(0); -var _A = 'A'.charCodeAt(0); -var _E = 'E'.charCodeAt(0); -var _F = 'F'.charCodeAt(0); -var _I = 'I'.charCodeAt(0); -var _L = 'L'.charCodeAt(0); -var _P = 'P'.charCodeAt(0); -var _R = 'R'.charCodeAt(0); -var _U = 'U'.charCodeAt(0); -var _X = 'X'.charCodeAt(0); -var _Z = 'Z'.charCodeAt(0); -var _0 = '0'.charCodeAt(0); -var _9 = '9'.charCodeAt(0); -var _TLD = '~'.charCodeAt(0); -var _HAT = '^'.charCodeAt(0); -var _EQS = '='.charCodeAt(0); -var _PIP = '|'.charCodeAt(0); -var _MIN = '-'.charCodeAt(0); -var _USC = '_'.charCodeAt(0); -var _PRC = '%'.charCodeAt(0); -var _MUL = '*'.charCodeAt(0); -var _LPA = '('.charCodeAt(0); -var _RPA = ')'.charCodeAt(0); -var _LAN = '<'.charCodeAt(0); -var _RAN = '>'.charCodeAt(0); -var _ATS = '@'.charCodeAt(0); -var _HSH = '#'.charCodeAt(0); -var _DLR = '$'.charCodeAt(0); -var _BSL = '\\'.charCodeAt(0); -var _FSL = '/'.charCodeAt(0); -var _NWL = '\n'.charCodeAt(0); -var _CAR = '\r'.charCodeAt(0); -var _LFD = '\f'.charCodeAt(0); -var _DQO = '"'.charCodeAt(0); -var _SQO = '\''.charCodeAt(0); -var _WSP = ' '.charCodeAt(0); -var _TAB = '\t'.charCodeAt(0); -var _SEM = ';'.charCodeAt(0); -var _COL = ':'.charCodeAt(0); -var _CUL = '{'.charCodeAt(0); -var _CUR = '}'.charCodeAt(0); -var _BRL = '['.charCodeAt(0); -var _BRR = ']'.charCodeAt(0); -var _CMA = ','.charCodeAt(0); -var _DOT = '.'.charCodeAt(0); -var _BNG = '!'.charCodeAt(0); - -var _url = [_u, _U, _r, _R, _l, _L, _LPA, _LPA]; -var _url_prefix = [_u, _U, _r, _R, _l, _L, _MIN, _MIN, _p, _P, _r, _R, _e, _E, _f, _F, _i, _I, _x, _X, _LPA, _LPA]; - -var staticTokenTable:{[code:number]:TokenType;} = {}; +const _a = 'a'.charCodeAt(0); +const _e = 'e'.charCodeAt(0); +const _f = 'f'.charCodeAt(0); +const _i = 'i'.charCodeAt(0); +const _l = 'l'.charCodeAt(0); +const _p = 'p'.charCodeAt(0); +const _r = 'r'.charCodeAt(0); +const _u = 'u'.charCodeAt(0); +const _x = 'x'.charCodeAt(0); +const _z = 'z'.charCodeAt(0); +const _A = 'A'.charCodeAt(0); +const _E = 'E'.charCodeAt(0); +const _F = 'F'.charCodeAt(0); +const _I = 'I'.charCodeAt(0); +const _L = 'L'.charCodeAt(0); +const _P = 'P'.charCodeAt(0); +const _R = 'R'.charCodeAt(0); +const _U = 'U'.charCodeAt(0); +const _X = 'X'.charCodeAt(0); +const _Z = 'Z'.charCodeAt(0); +const _0 = '0'.charCodeAt(0); +const _9 = '9'.charCodeAt(0); +const _TLD = '~'.charCodeAt(0); +const _HAT = '^'.charCodeAt(0); +const _EQS = '='.charCodeAt(0); +const _PIP = '|'.charCodeAt(0); +const _MIN = '-'.charCodeAt(0); +const _USC = '_'.charCodeAt(0); +const _PRC = '%'.charCodeAt(0); +const _MUL = '*'.charCodeAt(0); +const _LPA = '('.charCodeAt(0); +const _RPA = ')'.charCodeAt(0); +const _LAN = '<'.charCodeAt(0); +const _RAN = '>'.charCodeAt(0); +const _ATS = '@'.charCodeAt(0); +const _HSH = '#'.charCodeAt(0); +const _DLR = '$'.charCodeAt(0); +const _BSL = '\\'.charCodeAt(0); +const _FSL = '/'.charCodeAt(0); +const _NWL = '\n'.charCodeAt(0); +const _CAR = '\r'.charCodeAt(0); +const _LFD = '\f'.charCodeAt(0); +const _DQO = '"'.charCodeAt(0); +const _SQO = '\''.charCodeAt(0); +const _WSP = ' '.charCodeAt(0); +const _TAB = '\t'.charCodeAt(0); +const _SEM = ';'.charCodeAt(0); +const _COL = ':'.charCodeAt(0); +const _CUL = '{'.charCodeAt(0); +const _CUR = '}'.charCodeAt(0); +const _BRL = '['.charCodeAt(0); +const _BRR = ']'.charCodeAt(0); +const _CMA = ','.charCodeAt(0); +const _DOT = '.'.charCodeAt(0); +const _BNG = '!'.charCodeAt(0); + +const _url = [_u, _U, _r, _R, _l, _L, _LPA, _LPA]; +const _url_prefix = [_u, _U, _r, _R, _l, _L, _MIN, _MIN, _p, _P, _r, _R, _e, _E, _f, _F, _i, _I, _x, _X, _LPA, _LPA]; + +const staticTokenTable:{[code:number]:TokenType;} = {}; staticTokenTable[_SEM] = TokenType.SemiColon; staticTokenTable[_COL] = TokenType.Colon; staticTokenTable[_CUL] = TokenType.CurlyL; @@ -207,7 +208,7 @@ staticTokenTable[_LPA] = TokenType.ParenthesisL; staticTokenTable[_RPA] = TokenType.ParenthesisR; staticTokenTable[_CMA] = TokenType.Comma; -var staticUnitTable:{[code:number]:TokenType;} = {}; +const staticUnitTable:{[code:number]:TokenType;} = {}; staticUnitTable['em'] = TokenType.EMS; staticUnitTable['ex'] = TokenType.EXS; staticUnitTable['px'] = TokenType.Length; @@ -231,16 +232,19 @@ export class Scanner { public stream: MultiLineStream; public ignoreComment = true; + public ignoreWhitespace = true; public setSource(input: string): void { this.stream = new MultiLineStream(input); } - public finishToken(token: IToken, type: TokenType, text?: string): IToken { - token.len = this.stream.pos() - token.offset; - token.type = type; - token.text = text || this.stream.substring(token.offset); - return token; + public finishToken(offset: number, type: TokenType, text?: string): IToken { + return { + offset: offset, + len: this.stream.pos() - offset, + type: type, + text: text || this.stream.substring(offset) + }; } public substring(offset:number, len:number):string { @@ -255,71 +259,51 @@ export class Scanner { this.stream.goBackTo(pos); } - public scan(ignoreWhitespace:boolean=true): IToken { - - var result:IToken = { - type: undefined, - text: undefined, - offset: this.stream.pos(), - len: 0 - }; - - // Whitespace - if asked for - if (this._whitespace()) { - if (!ignoreWhitespace) { - return this.finishToken(result, TokenType.Whitespace); - } else { - return this.scan(ignoreWhitespace); - } + public scan(): IToken { + // processes all whitespaces and comments + let triviaToken = this.trivia(); + if (triviaToken !== null) { + return triviaToken; } - let tokenType: TokenType = void 0; - - // Comment - CSS - if (this._comment()) { - if (!this.ignoreComment) { - return this.finishToken(result, tokenType); - } else { - return this.scan(ignoreWhitespace); - } - } + let offset = this.stream.pos(); // End of file/input if (this.stream.eos()) { - return this.finishToken(result, TokenType.EOF); + return this.finishToken(offset, TokenType.EOF); } // CDO if (this.stream.advanceIfChars([_MIN, _MIN, _RAN])) { - return this.finishToken(result, TokenType.CDC); + return this.finishToken(offset, TokenType.CDC); } // URL - tokenType = this._url(); + let tokenType = this._url(); if (tokenType !== null) { - return this.finishToken(result, tokenType); + return this.finishToken(offset, tokenType); } - var content: string[] = []; + let content: string[] = []; if (this.ident(content)) { - return this.finishToken(result, TokenType.Ident, content.join('')); + return this.finishToken(offset, TokenType.Ident, content.join('')); } // at-keyword if (this.stream.advanceIfChar(_ATS)) { content = [ '@' ]; if (this.ident(content)) { - var keywordText = content.join(''); + let keywordText = content.join(''); if (keywordText === '@charset') { - return this.finishToken(result, TokenType.Charset, keywordText); + return this.finishToken(offset, TokenType.Charset, keywordText); } - return this.finishToken(result, TokenType.AtKeyword, keywordText); + return this.finishToken(offset, TokenType.AtKeyword, keywordText); } else { - return this.finishToken(result, TokenType.Delim); + return this.finishToken(offset, TokenType.Delim); } } @@ -327,93 +311,93 @@ export class Scanner { if (this.stream.advanceIfChar(_HSH)) { content = [ '#' ]; if (this._name(content)) { - return this.finishToken(result, TokenType.Hash, content.join('')); + return this.finishToken(offset, TokenType.Hash, content.join('')); } else { - return this.finishToken(result, TokenType.Delim); + return this.finishToken(offset, TokenType.Delim); } } // Important if (this.stream.advanceIfChar(_BNG)) { - return this.finishToken(result, TokenType.Exclamation); + return this.finishToken(offset, TokenType.Exclamation); } // Numbers if (this._number()) { - var pos = this.stream.pos(); - content = [ this.stream.substring(result.offset, pos) ]; + let pos = this.stream.pos(); + content = [ this.stream.substring(offset, pos) ]; if (this.stream.advanceIfChar(_PRC)) { // Percentage 43% - return this.finishToken(result, TokenType.Percentage); + return this.finishToken(offset, TokenType.Percentage); } else if (this.ident(content)) { let dim = this.stream.substring(pos).toLowerCase(); tokenType = staticUnitTable[dim]; if (typeof tokenType !== 'undefined') { // Known dimension 43px - return this.finishToken(result, tokenType, content.join('')); + return this.finishToken(offset, tokenType, content.join('')); } else { // Unknown dimension 43ft - return this.finishToken(result, TokenType.Dimension, content.join('')); + return this.finishToken(offset, TokenType.Dimension, content.join('')); } } - return this.finishToken(result, TokenType.Num); + return this.finishToken(offset, TokenType.Num); } // String, BadString content = []; tokenType = this._string(content); if (tokenType !== null) { - return this.finishToken(result, tokenType, content.join('')); + return this.finishToken(offset, tokenType, content.join('')); } // single character tokens tokenType = staticTokenTable[this.stream.peekChar()]; if (typeof tokenType !== 'undefined') { this.stream.advance(1); - return this.finishToken(result, tokenType); + return this.finishToken(offset, tokenType); } // includes ~= if (this.stream.peekChar(0) === _TLD && this.stream.peekChar(1) === _EQS) { this.stream.advance(2); - return this.finishToken(result, TokenType.Includes); + return this.finishToken(offset, TokenType.Includes); } // DashMatch |= if (this.stream.peekChar(0) === _PIP && this.stream.peekChar(1) === _EQS) { this.stream.advance(2); - return this.finishToken(result, TokenType.Dashmatch); + return this.finishToken(offset, TokenType.Dashmatch); } // Substring operator *= if (this.stream.peekChar(0) === _MUL && this.stream.peekChar(1) === _EQS) { this.stream.advance(2); - return this.finishToken(result, TokenType.SubstringOperator); + return this.finishToken(offset, TokenType.SubstringOperator); } // Substring operator ^= if (this.stream.peekChar(0) === _HAT && this.stream.peekChar(1) === _EQS) { this.stream.advance(2); - return this.finishToken(result, TokenType.PrefixOperator); + return this.finishToken(offset, TokenType.PrefixOperator); } // Substring operator $= if (this.stream.peekChar(0) === _DLR && this.stream.peekChar(1) === _EQS) { this.stream.advance(2); - return this.finishToken(result, TokenType.SuffixOperator); + return this.finishToken(offset, TokenType.SuffixOperator); } // Delim this.stream.nextChar(); - return this.finishToken(result, TokenType.Delim); + return this.finishToken(offset, TokenType.Delim); } private _matchWordAnyCase(characters:number[]): boolean { - var index = 0; + let index = 0; this.stream.advanceWhileChar((ch:number) => { - var result = characters[index] === ch || characters[index+1] === ch; + let result = characters[index] === ch || characters[index+1] === ch; if (result) { index += 2; } @@ -427,9 +411,26 @@ export class Scanner { } } - private _comment():boolean { + protected trivia(): IToken { + while (true) { + let offset = this.stream.pos(); + if (this._whitespace()) { + if (!this.ignoreWhitespace) { + return this.finishToken(offset, TokenType.Whitespace); + } + } else if (this.comment()) { + if (!this.ignoreComment) { + return this.finishToken(offset, TokenType.Comment); + } + } else { + return null; + } + } + } + + protected comment():boolean { if (this.stream.advanceIfChars([_FSL, _MUL])) { - var success = false, hot = false; + let success = false, hot = false; this.stream.advanceWhileChar((ch) => { if (hot && ch === _FSL) { success = true; @@ -448,7 +449,7 @@ export class Scanner { } private _number():boolean { - var npeek = 0, ch:number; + let npeek = 0, ch:number; if (this.stream.peekChar() === _DOT) { npeek = 1; } @@ -464,7 +465,7 @@ export class Scanner { } private _newline(result: string[]):boolean { - var ch = this.stream.peekChar(); + let ch = this.stream.peekChar(); switch (ch) { case _CAR: case _LFD: @@ -482,11 +483,11 @@ export class Scanner { } private _escape(result: string[], includeNewLines?:boolean):boolean { - var ch = this.stream.peekChar(); + let ch = this.stream.peekChar(); if (ch === _BSL) { this.stream.advance(1); ch = this.stream.peekChar(); - var hexNumCount = 0; + let hexNumCount = 0; while (hexNumCount < 6 && (ch >= _0 && ch <= _9 || ch >= _a && ch <= _f || ch >= _A && ch <= _F)) { this.stream.advance(1); ch = this.stream.peekChar(); @@ -494,7 +495,7 @@ export class Scanner { } if (hexNumCount > 0) { try { - var hexVal= parseInt(this.stream.substring(this.stream.pos() - hexNumCount), 16); + let hexVal= parseInt(this.stream.substring(this.stream.pos() - hexNumCount), 16); if (hexVal) { result.push(String.fromCharCode(hexVal)); } @@ -523,7 +524,7 @@ export class Scanner { private _stringChar(closeQuote: number, result: string[]) { // not closeQuote, not backslash, not newline - var ch = this.stream.peekChar(); + let ch = this.stream.peekChar(); if (ch !== 0 && ch !== closeQuote && ch !== _BSL && ch !== _CAR && ch !== _LFD && ch !== _NWL) { this.stream.advance(1); result.push(String.fromCharCode(ch)); @@ -534,7 +535,7 @@ export class Scanner { private _string(result: string[]):TokenType { if (this.stream.peekChar() === _SQO || this.stream.peekChar() === _DQO) { - var closeQuote = this.stream.nextChar(); + let closeQuote = this.stream.nextChar(); result.push(String.fromCharCode(closeQuote)); while (this._stringChar(closeQuote, result) || this._escape(result, true)) { @@ -555,7 +556,7 @@ export class Scanner { private _url():TokenType { if (this._matchWordAnyCase(_url) || this._matchWordAnyCase(_url_prefix)) { this._whitespace(); - var tokenType = TokenType.URI, stringType = this._string([]); + let tokenType = TokenType.URI, stringType = this._string([]); if (stringType === TokenType.BadString) { tokenType = TokenType.BadUri; @@ -577,14 +578,14 @@ export class Scanner { } private _whitespace():boolean { - var n = this.stream.advanceWhileChar((ch) => { + let n = this.stream.advanceWhileChar((ch) => { return ch === _WSP || ch === _TAB || ch === _NWL || ch === _LFD || ch === _CAR; }); return n > 0; } private _name(result:string[]):boolean { - var matched = false; + let matched = false; while (this._identChar(result) || this._escape(result)) { matched = true; } @@ -592,10 +593,10 @@ export class Scanner { } protected ident(result:string[]):boolean { - var pos = this.stream.pos(); - var hasMinus = this._minus(result); + let pos = this.stream.pos(); + let hasMinus = this._minus(result); if (hasMinus && this._minus(result) /* -- */) { - var hasContent = false; + let hasContent = false; while (this._identChar(result) || this._escape(result)) { hasContent = true; } @@ -613,7 +614,7 @@ export class Scanner { } private _identFirstChar(result:string[]):boolean { - var ch = this.stream.peekChar(); + let ch = this.stream.peekChar(); if (ch === _USC || // _ ch >= _a && ch <= _z || // a-z ch >= _A && ch <= _Z || // A-Z @@ -627,7 +628,7 @@ export class Scanner { private _minus(result:string[]):boolean { - var ch = this.stream.peekChar(); + let ch = this.stream.peekChar(); if (ch === _MIN) { this.stream.advance(1); result.push(String.fromCharCode(ch)); @@ -637,7 +638,7 @@ export class Scanner { } private _identChar(result:string[]):boolean { - var ch = this.stream.peekChar(); + let ch = this.stream.peekChar(); if (ch === _USC || // _ ch === _MIN || // - ch >= _a && ch <= _z || // a-z diff --git a/src/vs/languages/css/test/common/scanner.test.ts b/src/vs/languages/css/test/common/scanner.test.ts index 203e7266469..b4664664950 100644 --- a/src/vs/languages/css/test/common/scanner.test.ts +++ b/src/vs/languages/css/test/common/scanner.test.ts @@ -10,9 +10,9 @@ import Scanner = require('vs/languages/css/common/parser/cssScanner'); suite('CSS - Scanner', () => { - function assertSingleToken(scan: Scanner.Scanner, source: string, len: number, offset: number, text: string, type: Scanner.TokenType, ignoreWhitespace?:boolean):void { + function assertSingleToken(scan: Scanner.Scanner, source: string, len: number, offset: number, text: string, type: Scanner.TokenType):void { scan.setSource(source); - var token = scan.scan(ignoreWhitespace); + var token = scan.scan(); assert.equal(token.len, len); assert.equal(token.offset, offset); assert.equal(token.text, text); @@ -22,7 +22,17 @@ suite('CSS - Scanner', () => { test('Test Whitespace', function() { var scanner = new Scanner.Scanner(); assertSingleToken(scanner, ' @', 1, 1, '@', Scanner.TokenType.Delim); - assertSingleToken(scanner, ' @', 1, 0, ' ', Scanner.TokenType.Whitespace, false); + assertSingleToken(scanner, ' /* comment*/ \n/*comment*/@', 1, 26, '@', Scanner.TokenType.Delim); + + scanner = new Scanner.Scanner(); + scanner.ignoreWhitespace = false; + assertSingleToken(scanner, ' @', 1, 0, ' ', Scanner.TokenType.Whitespace); + assertSingleToken(scanner, '/*comment*/ @', 1, 11, ' ', Scanner.TokenType.Whitespace); + + scanner = new Scanner.Scanner(); + scanner.ignoreComment = false; + assertSingleToken(scanner, ' /*comment*/@', 11, 1, '/*comment*/', Scanner.TokenType.Comment); + assertSingleToken(scanner, '/*comment*/ @', 11, 0, '/*comment*/', Scanner.TokenType.Comment); }); test('Test Token Ident', function() { diff --git a/src/vs/languages/less/common/parser/lessScanner.ts b/src/vs/languages/less/common/parser/lessScanner.ts index 196ca531daa..037b17dc486 100644 --- a/src/vs/languages/less/common/parser/lessScanner.ts +++ b/src/vs/languages/less/common/parser/lessScanner.ts @@ -6,50 +6,44 @@ import scanner = require('vs/languages/css/common/parser/cssScanner'); -var _FSL = '/'.charCodeAt(0); -var _NWL = '\n'.charCodeAt(0); -var _CAR = '\r'.charCodeAt(0); -var _LFD = '\f'.charCodeAt(0); -var _TIC = '`'.charCodeAt(0); -var _DOT = '.'.charCodeAt(0); +const _FSL = '/'.charCodeAt(0); +const _NWL = '\n'.charCodeAt(0); +const _CAR = '\r'.charCodeAt(0); +const _LFD = '\f'.charCodeAt(0); +const _TIC = '`'.charCodeAt(0); +const _DOT = '.'.charCodeAt(0); -var customTokenValue = scanner.TokenType.CustomToken; -export var Ellipsis: scanner.TokenType = customTokenValue++; +let customTokenValue = scanner.TokenType.CustomToken; +export const Ellipsis: scanner.TokenType = customTokenValue++; export class LessScanner extends scanner.Scanner { - public scan(ignoreWhitespace:boolean=true): scanner.IToken { + public scan(): scanner.IToken { - var result:scanner.IToken = { - type: undefined, - text: undefined, - offset: this.stream.pos(), - len: 0 - }; - - // SingleLine Comments - if (this.lessComment()) { - if (!this.ignoreComment) { - return this.finishToken(result, scanner.TokenType.SingleLineComment); - } else { - return this.scan(ignoreWhitespace); - } + let triviaToken = this.trivia(); + if (triviaToken !== null) { + return triviaToken; } - // LESS: escaped JavaScript code `var a = "dddd"` - var tokenType = this.escapedJavaScript(); - if(tokenType !== null) { - return this.finishToken(result, tokenType); + let offset = this.stream.pos(); + + // LESS: escaped JavaScript code `let a = "dddd"` + let tokenType = this.escapedJavaScript(); + if (tokenType !== null) { + return this.finishToken(offset, tokenType); } - if(this.stream.advanceIfChars([_DOT, _DOT, _DOT])) { - return this.finishToken(result, Ellipsis); + if (this.stream.advanceIfChars([_DOT, _DOT, _DOT])) { + return this.finishToken(offset, Ellipsis); } - return super.scan(ignoreWhitespace); + return super.scan(); } - private lessComment():boolean { + protected comment():boolean { + if (super.comment()) { + return true; + } if (this.stream.advanceIfChars([_FSL, _FSL])) { this.stream.advanceWhileChar((ch:number) => { switch(ch) { @@ -68,8 +62,8 @@ export class LessScanner extends scanner.Scanner { } private escapedJavaScript():scanner.TokenType { - var ch = this.stream.peekChar(); - if(ch === _TIC) { + let ch = this.stream.peekChar(); + if (ch === _TIC) { this.stream.advance(1); this.stream.advanceWhileChar((ch) => { return ch !== _TIC; }); return this.stream.advanceIfChar(_TIC) ? scanner.TokenType.EscapedJavaScript : scanner.TokenType.BadEscapedJavaScript; diff --git a/src/vs/languages/less/test/common/scanner.test.ts b/src/vs/languages/less/test/common/scanner.test.ts index b000c39bc15..1f11d3f14e3 100644 --- a/src/vs/languages/less/test/common/scanner.test.ts +++ b/src/vs/languages/less/test/common/scanner.test.ts @@ -9,10 +9,10 @@ import assert = require('assert'); import scanner = require('vs/languages/css/common/parser/cssScanner'); import lessScanner = require('vs/languages/less/common/parser/lessScanner'); -function assertSingleToken(source: string, len: number, offset: number, text: string, type: scanner.TokenType, ignoreWhitespace?:boolean):void { +function assertSingleToken(source: string, len: number, offset: number, text: string, type: scanner.TokenType):void { var scan = new lessScanner.LessScanner(); scan.setSource(source); - var token = scan.scan(ignoreWhitespace); + var token = scan.scan(); assert.equal(token.len, len); assert.equal(token.offset, offset); assert.equal(token.text, text); @@ -35,5 +35,6 @@ suite('LESS - Scanner', () => { assertSingleToken('//this is a comment test', 0, 24, '', scanner.TokenType.EOF); assertSingleToken('// this is a comment test', 0, 25, '', scanner.TokenType.EOF); assertSingleToken('// this is a\na', 1, 13, 'a', scanner.TokenType.Ident); + assertSingleToken('// this is a\n// more\n \n/* comment */a', 1, 38, 'a', scanner.TokenType.Ident); }); }); diff --git a/src/vs/languages/sass/common/parser/sassScanner.ts b/src/vs/languages/sass/common/parser/sassScanner.ts index af24de30359..76ef3a73128 100644 --- a/src/vs/languages/sass/common/parser/sassScanner.ts +++ b/src/vs/languages/sass/common/parser/sassScanner.ts @@ -6,101 +6,96 @@ import scanner = require('vs/languages/css/common/parser/cssScanner'); -var _FSL = '/'.charCodeAt(0); -var _NWL = '\n'.charCodeAt(0); -var _CAR = '\r'.charCodeAt(0); -var _LFD = '\f'.charCodeAt(0); - -var _DLR = '$'.charCodeAt(0); -var _HSH = '#'.charCodeAt(0); -var _CUL = '{'.charCodeAt(0); -var _EQS = '='.charCodeAt(0); -var _BNG = '!'.charCodeAt(0); -var _LAN = '<'.charCodeAt(0); -var _RAN = '>'.charCodeAt(0); -var _DOT = '.'.charCodeAt(0); - -var customTokenValue = scanner.TokenType.CustomToken; - -export var VariableName = customTokenValue++; -export var InterpolationFunction: scanner.TokenType = customTokenValue++; -export var Default: scanner.TokenType = customTokenValue++; -export var EqualsOperator: scanner.TokenType = customTokenValue++; -export var NotEqualsOperator: scanner.TokenType = customTokenValue++; -export var GreaterEqualsOperator: scanner.TokenType = customTokenValue++; -export var SmallerEqualsOperator: scanner.TokenType = customTokenValue++; -export var Ellipsis: scanner.TokenType = customTokenValue++; +const _FSL = '/'.charCodeAt(0); +const _NWL = '\n'.charCodeAt(0); +const _CAR = '\r'.charCodeAt(0); +const _LFD = '\f'.charCodeAt(0); + +const _DLR = '$'.charCodeAt(0); +const _HSH = '#'.charCodeAt(0); +const _CUL = '{'.charCodeAt(0); +const _EQS = '='.charCodeAt(0); +const _BNG = '!'.charCodeAt(0); +const _LAN = '<'.charCodeAt(0); +const _RAN = '>'.charCodeAt(0); +const _DOT = '.'.charCodeAt(0); + +let customTokenValue = scanner.TokenType.CustomToken; + +export const VariableName = customTokenValue++; +export const InterpolationFunction: scanner.TokenType = customTokenValue++; +export const Default: scanner.TokenType = customTokenValue++; +export const EqualsOperator: scanner.TokenType = customTokenValue++; +export const NotEqualsOperator: scanner.TokenType = customTokenValue++; +export const GreaterEqualsOperator: scanner.TokenType = customTokenValue++; +export const SmallerEqualsOperator: scanner.TokenType = customTokenValue++; +export const Ellipsis: scanner.TokenType = customTokenValue++; export class SassScanner extends scanner.Scanner { - public scan(ignoreWhitespace:boolean=true): scanner.IToken { + public scan(): scanner.IToken { - var result:scanner.IToken = { - type: undefined, - text: undefined, - offset: this.stream.pos(), - len: 0 - }; - - // SingleLine Comments - if (this.sassComment()) { - if (!this.ignoreComment) { - return this.finishToken(result, scanner.TokenType.SingleLineComment); - } else { - return this.scan(ignoreWhitespace); - } + // processes all whitespaces and comments + const triviaToken = this.trivia(); + if (triviaToken !== null) { + return triviaToken; } + const offset = this.stream.pos(); + // sass variable if (this.stream.advanceIfChar(_DLR)) { - var content = [ '$' ]; + const content = [ '$' ]; if (this.ident(content)) { - return this.finishToken(result, VariableName, content.join('')); + return this.finishToken(offset, VariableName, content.join('')); } else { - this.stream.goBackTo(result.offset); + this.stream.goBackTo(offset); } } // Sass: interpolation function #{..}) if (this.stream.advanceIfChars([_HSH, _CUL])) { - return this.finishToken(result, InterpolationFunction); + return this.finishToken(offset, InterpolationFunction); } // operator == if (this.stream.advanceIfChars([_EQS, _EQS])) { - return this.finishToken(result, EqualsOperator); + return this.finishToken(offset, EqualsOperator); } // operator != if (this.stream.advanceIfChars([_BNG, _EQS])) { - return this.finishToken(result, NotEqualsOperator); + return this.finishToken(offset, NotEqualsOperator); } // operators <, <= if (this.stream.advanceIfChar(_LAN)) { if (this.stream.advanceIfChar(_EQS)) { - return this.finishToken(result, SmallerEqualsOperator); + return this.finishToken(offset, SmallerEqualsOperator); } - return this.finishToken(result, scanner.TokenType.Delim); + return this.finishToken(offset, scanner.TokenType.Delim); } // ooperators >, >= if (this.stream.advanceIfChar(_RAN)) { if (this.stream.advanceIfChar(_EQS)) { - return this.finishToken(result, GreaterEqualsOperator); + return this.finishToken(offset, GreaterEqualsOperator); } - return this.finishToken(result, scanner.TokenType.Delim); + return this.finishToken(offset, scanner.TokenType.Delim); } // ellipis if (this.stream.advanceIfChars([_DOT, _DOT, _DOT])) { - return this.finishToken(result, Ellipsis); + return this.finishToken(offset, Ellipsis); } - return super.scan(ignoreWhitespace); + return super.scan(); } - private sassComment():boolean { + protected comment():boolean { + if (super.comment()) { + return true; + } if (this.stream.advanceIfChars([_FSL, _FSL])) { this.stream.advanceWhileChar((ch:number) => { switch(ch) { -- GitLab