Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
掘金者说
vscode
提交
9e05f824
V
vscode
项目概览
掘金者说
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9e05f824
编写于
10月 07, 2016
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Remove unused methods from LineStream
上级
d65cbf44
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
42 addition
and
842 deletion
+42
-842
src/vs/editor/common/modes/lineStream.ts
src/vs/editor/common/modes/lineStream.ts
+1
-493
src/vs/editor/test/common/model/model.modes.test.ts
src/vs/editor/test/common/model/model.modes.test.ts
+7
-6
src/vs/editor/test/common/modes/lineStream.test.ts
src/vs/editor/test/common/modes/lineStream.test.ts
+8
-335
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
+3
-1
src/vs/editor/test/common/modes/tokenization.test.ts
src/vs/editor/test/common/modes/tokenization.test.ts
+23
-7
未找到文件。
src/vs/editor/common/modes/lineStream.ts
浏览文件 @
9e05f824
...
...
@@ -4,33 +4,6 @@
*--------------------------------------------------------------------------------------------*/
'
use strict
'
;
import
{
CharacterClassifier
}
from
'
vs/editor/common/core/characterClassifier
'
;
class
CharacterSet
{
private
static
_CACHE
:{
[
key
:
string
]:
CharacterSet
;
}
=
{};
// TODO@Alex unbounded cache
public
static
getOrCreate
(
source
:
string
):
CharacterSet
{
if
(
!
CharacterSet
.
_CACHE
.
hasOwnProperty
(
source
))
{
CharacterSet
.
_CACHE
[
source
]
=
new
CharacterSet
(
source
);
}
return
CharacterSet
.
_CACHE
[
source
];
}
private
_classifier
:
CharacterClassifier
<
boolean
>
;
constructor
(
source
:
string
)
{
this
.
_classifier
=
new
CharacterClassifier
<
boolean
>
(
false
);
for
(
let
i
=
0
,
len
=
source
.
length
;
i
<
len
;
i
++
)
{
this
.
_classifier
.
set
(
source
.
charCodeAt
(
i
),
true
);
}
}
public
contains
(
charCode
:
number
):
boolean
{
return
this
.
_classifier
.
get
(
charCode
);
}
}
/**
* A LineStream is a character & token stream abstraction over a line of text. It
* is never multi-line. The stream can be navigated character by character, or
...
...
@@ -42,23 +15,11 @@ export class LineStream {
private
_source
:
string
;
private
_sourceLength
:
number
;
private
_pos
:
number
;
private
_whitespace
:
string
;
private
_whitespaceArr
:
CharacterSet
;
private
_separators
:
string
;
private
_separatorsArr
:
CharacterSet
;
private
_tokenStart
:
number
;
private
_tokenEnd
:
number
;
constructor
(
source
:
string
)
{
this
.
_source
=
source
;
this
.
_sourceLength
=
source
.
length
;
this
.
_pos
=
0
;
this
.
_whitespace
=
'
\t
\
u00a0
'
;
this
.
_whitespaceArr
=
CharacterSet
.
getOrCreate
(
this
.
_whitespace
);
this
.
_separators
=
''
;
this
.
_separatorsArr
=
CharacterSet
.
getOrCreate
(
this
.
_separators
);
this
.
_tokenStart
=
-
1
;
this
.
_tokenEnd
=
-
1
;
}
/**
...
...
@@ -86,61 +47,14 @@ export class LineStream {
return
this
.
_source
[
this
.
_pos
];
}
/**
* Returns the next character in the stream, and advances it by one character.
*/
public
next
():
string
{
// Check EOS
if
(
this
.
_pos
>=
this
.
_sourceLength
)
{
throw
new
Error
(
'
Stream is at the end
'
);
}
// Reset peeked token
this
.
_tokenStart
=
-
1
;
this
.
_tokenEnd
=
-
1
;
return
this
.
_source
[
this
.
_pos
++
];
}
/**
* Advances the stream by one character.
*/
public
next2
():
void
{
// Check EOS
if
(
this
.
_pos
>=
this
.
_sourceLength
)
{
throw
new
Error
(
'
Stream is at the end
'
);
}
// Reset peeked token
this
.
_tokenStart
=
-
1
;
this
.
_tokenEnd
=
-
1
;
this
.
_pos
++
;
}
/**
* Advances the stream by `n` characters.
*/
public
advance
(
n
:
number
):
string
{
if
(
n
===
0
)
{
return
''
;
}
const
oldPos
=
this
.
_pos
;
this
.
_pos
+=
n
;
// Reset peeked token
this
.
_tokenStart
=
-
1
;
this
.
_tokenEnd
=
-
1
;
return
this
.
_source
.
substring
(
oldPos
,
this
.
_pos
);
}
private
_advance2
(
n
:
number
):
number
{
public
advance
(
n
:
number
):
number
{
if
(
n
===
0
)
{
return
n
;
}
this
.
_pos
+=
n
;
// Reset peeked token
this
.
_tokenStart
=
-
1
;
this
.
_tokenEnd
=
-
1
;
return
n
;
}
...
...
@@ -150,7 +64,6 @@ export class LineStream {
public
advanceToEOS
():
string
{
const
oldPos
=
this
.
_pos
;
this
.
_pos
=
this
.
_sourceLength
;
this
.
resetPeekedToken
();
return
this
.
_source
.
substring
(
oldPos
,
this
.
_pos
);
}
...
...
@@ -159,410 +72,5 @@ export class LineStream {
*/
public
goBack
(
n
:
number
)
{
this
.
_pos
-=
n
;
this
.
resetPeekedToken
();
}
private
createPeeker
(
condition
:
RegExp
|
string
):()
=>
number
{
if
(
condition
instanceof
RegExp
)
{
return
()
=>
{
let
result
=
condition
.
exec
(
this
.
_source
.
substr
(
this
.
_pos
));
if
(
result
===
null
)
{
return
0
;
}
else
if
(
result
.
index
!==
0
)
{
throw
new
Error
(
'
Regular expression must begin with the character "^"
'
);
}
return
result
[
0
].
length
;
};
}
else
if
((
typeof
condition
===
'
string
'
)
&&
condition
)
{
return
()
=>
{
const
len
=
condition
.
length
;
let
match
=
(
this
.
_pos
+
len
<=
this
.
_sourceLength
);
for
(
let
i
=
0
;
match
&&
i
<
len
;
i
++
)
{
match
=
this
.
_source
.
charCodeAt
(
this
.
_pos
+
i
)
===
condition
.
charCodeAt
(
i
);
}
return
(
match
?
len
:
0
);
};
}
throw
new
Error
(
'
Condition must be either a regular expression, function or a non-empty string
'
);
}
// --- BEGIN `_advanceIfStringCaseInsensitive`
private
_advanceIfStringCaseInsensitive
(
condition
:
string
):
number
{
const
oldPos
=
this
.
_pos
;
const
source
=
this
.
_source
;
const
len
=
condition
.
length
;
if
(
len
<
1
||
oldPos
+
len
>
this
.
_sourceLength
)
{
return
0
;
}
for
(
let
i
=
0
;
i
<
len
;
i
++
)
{
if
(
source
.
charAt
(
oldPos
+
i
).
toLowerCase
()
!==
condition
.
charAt
(
i
).
toLowerCase
())
{
return
0
;
}
}
return
len
;
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public
advanceIfStringCaseInsensitive
(
condition
:
string
):
string
{
return
this
.
advance
(
this
.
_advanceIfStringCaseInsensitive
(
condition
));
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public
advanceIfStringCaseInsensitive2
(
condition
:
string
):
number
{
return
this
.
_advance2
(
this
.
_advanceIfStringCaseInsensitive
(
condition
));
}
// --- END
// --- BEGIN `advanceIfString`
private
_advanceIfString
(
condition
:
string
):
number
{
const
oldPos
=
this
.
_pos
;
const
source
=
this
.
_source
;
const
len
=
condition
.
length
;
if
(
len
<
1
||
oldPos
+
len
>
this
.
_sourceLength
)
{
return
0
;
}
for
(
let
i
=
0
;
i
<
len
;
i
++
)
{
if
(
source
.
charCodeAt
(
oldPos
+
i
)
!==
condition
.
charCodeAt
(
i
))
{
return
0
;
}
}
return
len
;
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public
advanceIfString
(
condition
:
string
):
string
{
return
this
.
advance
(
this
.
_advanceIfString
(
condition
));
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public
advanceIfString2
(
condition
:
string
):
number
{
return
this
.
_advance2
(
this
.
_advanceIfString
(
condition
));
}
// --- END
// --- BEGIN `advanceIfString`
private
_advanceIfCharCode
(
charCode
:
number
):
number
{
if
(
this
.
_pos
<
this
.
_sourceLength
&&
this
.
_source
.
charCodeAt
(
this
.
_pos
)
===
charCode
)
{
return
1
;
}
return
0
;
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public
advanceIfCharCode
(
charCode
:
number
):
string
{
return
this
.
advance
(
this
.
_advanceIfCharCode
(
charCode
));
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public
advanceIfCharCode2
(
charCode
:
number
):
number
{
return
this
.
_advance2
(
this
.
_advanceIfCharCode
(
charCode
));
}
// --- END
// --- BEGIN `advanceIfRegExp`
private
_advanceIfRegExp
(
condition
:
RegExp
):
number
{
if
(
this
.
_pos
>=
this
.
_sourceLength
)
{
return
0
;
}
if
(
!
condition
.
test
(
this
.
_source
.
substr
(
this
.
_pos
)))
{
return
0
;
}
return
RegExp
.
lastMatch
.
length
;
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public
advanceIfRegExp
(
condition
:
RegExp
):
string
{
return
this
.
advance
(
this
.
_advanceIfRegExp
(
condition
));
}
/**
* Advances the stream if the next characters validate a condition. A condition can be
*
* - a regular expression (always starting with ^)
* EXAMPLES: /^\d+/, /^function|var|interface|class/
*
* - a string
* EXAMPLES: "1954", "albert"
*/
public
advanceIfRegExp2
(
condition
:
RegExp
):
number
{
return
this
.
_advance2
(
this
.
_advanceIfRegExp
(
condition
));
}
// --- END
private
advanceLoop
(
condition
:
RegExp
|
string
,
isWhile
:
boolean
,
including
:
boolean
):
string
{
if
(
this
.
eos
())
{
return
''
;
}
const
peeker
=
this
.
createPeeker
(
condition
);
const
oldPos
=
this
.
_pos
;
let
n
=
0
;
let
f
=
null
;
if
(
isWhile
)
{
f
=
(
n
)
=>
{
return
n
>
0
;
};
}
else
{
f
=
(
n
)
=>
{
return
n
===
0
;
};
}
while
(
!
this
.
eos
()
&&
f
(
n
=
peeker
()))
{
if
(
n
>
0
)
{
this
.
advance
(
n
);
}
else
{
this
.
next
();
}
}
if
(
including
&&
!
this
.
eos
())
{
this
.
advance
(
n
);
}
return
this
.
_source
.
substring
(
oldPos
,
this
.
_pos
);
}
/**
* Advances the stream while the next characters validate a condition. Check #advanceIf for
* details on the possible types for condition.
*/
public
advanceWhile
(
condition
:
RegExp
|
string
):
string
{
return
this
.
advanceLoop
(
condition
,
true
,
false
);
}
/**
* Advances the stream until the some characters validate a condition. Check #advanceIf for
* details on the possible types for condition. The `including` boolean value indicates
* whether the stream will advance the characters that matched the condition as well, or not.
*/
public
advanceUntil
(
condition
:
RegExp
|
string
,
including
:
boolean
):
string
{
return
this
.
advanceLoop
(
condition
,
false
,
including
);
}
// --- BEGIN `advanceUntilString`
private
_advanceUntilString
(
condition
:
string
,
including
:
boolean
):
number
{
if
(
this
.
eos
()
||
condition
.
length
===
0
)
{
return
0
;
}
const
oldPos
=
this
.
_pos
;
const
index
=
this
.
_source
.
indexOf
(
condition
,
oldPos
);
if
(
index
===
-
1
)
{
// String was not found => advanced to `eos`
return
(
this
.
_sourceLength
-
oldPos
);
}
if
(
including
)
{
// String was found => advance to include `condition`
return
(
index
+
condition
.
length
-
oldPos
);
}
// String was found => advance right before `condition`
return
(
index
-
oldPos
);
}
public
advanceUntilString
(
condition
:
string
,
including
:
boolean
):
string
{
return
this
.
advance
(
this
.
_advanceUntilString
(
condition
,
including
));
}
public
advanceUntilString2
(
condition
:
string
,
including
:
boolean
):
number
{
return
this
.
_advance2
(
this
.
_advanceUntilString
(
condition
,
including
));
}
// --- END
private
resetPeekedToken
()
{
this
.
_tokenStart
=
-
1
;
this
.
_tokenEnd
=
-
1
;
}
/**
* The token rules define how consecutive characters should be put together as a token,
* or separated into two different tokens. They are given through a separator characters
* string and a whitespace characters string. A separator is always one token. Consecutive
* whitespace is always one token. Everything in between these two token types, is also a token.
*
* EXAMPLE: stream.setTokenRules("+-", " ");
* Setting these token rules defines the tokens for the string "123+456 - 7" as being
* ["123", "+", "456", " ", "-", " ", "7"]
*/
public
setTokenRules
(
separators
:
string
,
whitespace
:
string
):
void
{
if
(
this
.
_separators
!==
separators
||
this
.
_whitespace
!==
whitespace
)
{
this
.
_separators
=
separators
;
this
.
_separatorsArr
=
CharacterSet
.
getOrCreate
(
this
.
_separators
);
this
.
_whitespace
=
whitespace
;
this
.
_whitespaceArr
=
CharacterSet
.
getOrCreate
(
this
.
_whitespace
);
this
.
resetPeekedToken
();
}
}
// --- tokens
/**
* Returns the next token, given that the stream was configured with token rules.
*/
public
peekToken
():
string
{
if
(
this
.
_tokenStart
!==
-
1
)
{
return
this
.
_source
.
substring
(
this
.
_tokenStart
,
this
.
_tokenEnd
);
}
const
source
=
this
.
_source
;
const
sourceLength
=
this
.
_sourceLength
;
const
whitespaceArr
=
this
.
_whitespaceArr
;
const
separatorsArr
=
this
.
_separatorsArr
;
let
tokenStart
=
this
.
_pos
;
// Check EOS
if
(
tokenStart
>=
sourceLength
)
{
throw
new
Error
(
'
Stream is at the end
'
);
}
// Skip whitespace
while
(
whitespaceArr
.
contains
(
source
.
charCodeAt
(
tokenStart
))
&&
tokenStart
<
sourceLength
)
{
tokenStart
++
;
}
let
tokenEnd
=
tokenStart
;
// If a separator is hit, it is a token
if
(
separatorsArr
.
contains
(
source
.
charCodeAt
(
tokenEnd
))
&&
tokenEnd
<
sourceLength
)
{
tokenEnd
++
;
}
else
{
// Advance until a separator or a whitespace is hit
while
(
!
separatorsArr
.
contains
(
source
.
charCodeAt
(
tokenEnd
))
&&
!
whitespaceArr
.
contains
(
source
.
charCodeAt
(
tokenEnd
))
&&
tokenEnd
<
sourceLength
)
{
tokenEnd
++
;
}
}
// Cache peeked token
this
.
_tokenStart
=
tokenStart
;
this
.
_tokenEnd
=
tokenEnd
;
return
source
.
substring
(
tokenStart
,
tokenEnd
);
}
/**
* Returns the next token, given that the stream was configured with token rules, and advances the
* stream by the exact length of the found token.
*/
public
nextToken
():
string
{
// Check EOS
if
(
this
.
_pos
>=
this
.
_sourceLength
)
{
throw
new
Error
(
'
Stream is at the end
'
);
}
// Peek token if necessary
let
result
:
string
;
if
(
this
.
_tokenStart
===
-
1
)
{
result
=
this
.
peekToken
();
}
else
{
result
=
this
.
_source
.
substring
(
this
.
_tokenStart
,
this
.
_tokenEnd
);
}
// Advance to tokenEnd
this
.
_pos
=
this
.
_tokenEnd
;
// Reset peeked token
this
.
_tokenStart
=
-
1
;
this
.
_tokenEnd
=
-
1
;
return
result
;
}
// -- whitespace
/**
* Returns the next whitespace, if found. Returns an empty string otherwise.
*/
public
peekWhitespace
():
string
{
const
source
=
this
.
_source
;
const
sourceLength
=
this
.
_sourceLength
;
const
whitespaceArr
=
this
.
_whitespaceArr
;
let
peek
=
this
.
_pos
;
while
(
whitespaceArr
.
contains
(
source
.
charCodeAt
(
peek
))
&&
peek
<
sourceLength
)
{
peek
++
;
}
return
source
.
substring
(
this
.
_pos
,
peek
);
}
// --- BEGIN `skipWhitespace`
private
_skipWhitespace
():
number
{
const
source
=
this
.
_source
;
const
sourceLength
=
this
.
_sourceLength
;
const
whitespaceArr
=
this
.
_whitespaceArr
;
const
oldPos
=
this
.
_pos
;
let
peek
=
this
.
_pos
;
while
(
whitespaceArr
.
contains
(
source
.
charCodeAt
(
peek
))
&&
peek
<
sourceLength
)
{
peek
++
;
}
return
(
peek
-
oldPos
);
}
/**
* Returns the next whitespace, if found, and advances the stream by the exact length of the found
* whitespace. Returns an empty string otherwise.
*/
public
skipWhitespace
():
string
{
return
this
.
advance
(
this
.
_skipWhitespace
());
}
/**
* Returns the next whitespace, if found, and advances the stream by the exact length of the found
* whitespace. Returns an empty string otherwise.
*/
public
skipWhitespace2
():
number
{
return
this
.
_advance2
(
this
.
_skipWhitespace
());
}
// --- END
}
src/vs/editor/test/common/model/model.modes.test.ts
浏览文件 @
9e05f824
...
...
@@ -40,7 +40,9 @@ suite('Editor Model - Model Modes 1', () => {
return
this
===
other
;
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
calledState
.
calledFor
.
push
(
stream
.
next
());
let
chr
=
stream
.
peek
();
stream
.
advance
(
1
);
calledState
.
calledFor
.
push
(
chr
);
stream
.
advanceToEOS
();
return
{
type
:
''
};
}
...
...
@@ -190,10 +192,7 @@ suite('Editor Model - Model Modes 2', () => {
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
var
line
=
''
;
while
(
!
stream
.
eos
())
{
line
+=
stream
.
next
();
}
var
line
=
stream
.
advanceToEOS
();
this
.
prevLineContent
=
line
;
return
{
type
:
''
};
}
...
...
@@ -329,7 +328,9 @@ suite('Editor Model - Token Iterator', () => {
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
var
ndash
=
this
.
n
,
value
=
''
;
while
(
!
stream
.
eos
()
&&
ndash
>
0
)
{
value
+=
stream
.
next
();
let
chr
=
stream
.
peek
();
stream
.
advance
(
1
);
value
+=
chr
;
ndash
--
;
}
return
{
type
:
'
n-
'
+
(
this
.
n
-
ndash
)
+
'
-
'
+
value
};
...
...
src/vs/editor/test/common/modes/lineStream.test.ts
浏览文件 @
9e05f824
...
...
@@ -8,361 +8,34 @@ import {LineStream} from 'vs/editor/common/modes/lineStream';
suite
(
'
Editor Modes - LineStream
'
,
()
=>
{
test
(
'
advanceIf - regex
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
...xxx...x.
'
);
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
'
x
'
);
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
'
x
'
);
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
'
x
'
);
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
'
x
'
);
assert
.
equal
(
lineStream
.
advanceIfRegExp
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
advanceWhile - regex
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
...xxx...x.
'
);
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
'
xxx
'
);
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
'
x
'
);
assert
.
equal
(
lineStream
.
advanceWhile
(
/^x/
),
''
);
lineStream
.
next
();
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
advanceUntil - regex
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
...x..xx..x
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
false
),
'
...
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
false
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
false
),
'
..
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
false
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
false
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
false
),
'
..
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
false
),
''
);
lineStream
.
next
();
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
advanceUntil - regex (including)
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
...x..xx..x
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
true
),
'
...x
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
true
),
'
..x
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
true
),
'
x
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
/^x/
,
true
),
'
..x
'
);
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
advanceIf - string
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
...abcabcabc...abc.
'
);
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
'
abc
'
);
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
'
abc
'
);
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
'
abc
'
);
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
'
abc
'
);
assert
.
equal
(
lineStream
.
advanceIfString
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
advanceWhile - string
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
...abcabcabc...abc.
'
);
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
'
abcabcabc
'
);
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
'
abc
'
);
assert
.
equal
(
lineStream
.
advanceWhile
(
'
abc
'
),
''
);
lineStream
.
next
();
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
advanceUntil - string
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
...abc..ab..abc..bc
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
'
abc
'
,
false
),
'
...
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
'
abc
'
,
false
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceUntil
(
'
abc
'
,
false
),
'
bc..ab..
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
'
abc
'
,
false
),
''
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
advanceUntil
(
'
abc
'
,
false
),
'
bc..bc
'
);
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
advanceUntil - string (including)
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
...abc..ab..abc..bc
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
'
abc
'
,
true
),
'
...abc
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
'
abc
'
,
true
),
'
..ab..abc
'
);
assert
.
equal
(
lineStream
.
advanceUntil
(
'
abc
'
,
true
),
'
..bc
'
);
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
skipWhitespace
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
\t
a bc d
\t
e
'
);
assert
.
equal
(
lineStream
.
skipWhitespace
(),
'
\t
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
skipWhitespace
(),
'
'
);
lineStream
.
next
();
lineStream
.
next
();
assert
.
equal
(
lineStream
.
skipWhitespace
(),
'
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
skipWhitespace
(),
'
\t
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
skipWhitespace
(),
'
'
);
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
peekToken
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
a b c edf
'
);
assert
.
equal
(
lineStream
.
peekToken
(),
'
a
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
peekToken
(),
'
b
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
peekToken
(),
'
b
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
peekToken
(),
'
c
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
peekToken
(),
'
c
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
peekToken
(),
'
c
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
peekToken
(),
'
edf
'
);
lineStream
.
next
();
assert
.
equal
(
lineStream
.
peekToken
(),
'
edf
'
);
lineStream
.
next
();
lineStream
.
next
();
lineStream
.
next
();
lineStream
.
next
();
assert
.
throws
(()
=>
{
lineStream
.
peekToken
();
});
assert
.
ok
(
lineStream
.
eos
());
});
test
(
'
nextToken
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
a b c edf
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
a
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
b
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
c
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
edf
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
''
);
assert
.
throws
(()
=>
{
lineStream
.
nextToken
();
});
assert
.
ok
(
lineStream
.
eos
());
});
function
newTokenStream
(
source
,
separators
,
whitespace
)
{
var
lineStream
=
new
LineStream
(
source
);
lineStream
.
setTokenRules
(
separators
,
whitespace
);
return
lineStream
;
}
function
checkPos
(
lineStream
,
pos
)
{
assert
.
equal
(
lineStream
.
pos
(),
pos
);
}
function
check
(
lineStream
,
pos
,
token
)
{
checkPos
(
lineStream
,
pos
);
assert
.
equal
(
lineStream
.
nextToken
(),
token
);
}
test
(
'
corner cases
'
,
()
=>
{
var
input
,
lineStream
;
var
noTokens
=
(
lineStream
)
=>
{
let
noTokens
=
(
lineStream
)
=>
{
assert
.
equal
(
lineStream
.
pos
(),
0
);
assert
.
ok
(
lineStream
.
eos
());
};
noTokens
(
newTokenStream
(
''
,
''
,
''
));
noTokens
(
newTokenStream
(
''
,
''
,
'
x
'
));
noTokens
(
newTokenStream
(
''
,
'
x
'
,
''
));
noTokens
(
newTokenStream
(
''
,
'
x
'
,
'
x
'
));
input
=
'
.....
'
;
lineStream
=
newTokenStream
(
input
,
'
.
'
,
''
);
for
(
var
i
=
0
;
i
<
input
.
length
;
i
++
)
{
check
(
lineStream
,
i
,
'
.
'
);
}
input
=
'
. . . . .
'
;
lineStream
=
newTokenStream
(
input
,
'
.
'
,
'
'
);
for
(
var
i
=
0
;
i
<
input
.
length
/
2
;
i
++
)
{
check
(
lineStream
,
(
i
*
2
)
,
'
.
'
);
}
input
=
'
. . . . .
'
;
lineStream
=
newTokenStream
(
input
,
'
.
'
,
'
'
);
for
(
var
i
=
0
;
i
<
input
.
length
/
2
;
i
++
)
{
check
(
lineStream
,
i
===
0
?
0
:
(
i
*
2
)
-
1
,
'
.
'
);
}
noTokens
(
new
LineStream
(
''
));
});
test
(
'
javascript assign
'
,
()
=>
{
var
lineStream
=
new
TokenStream
(
'
var foo =bar("foo"); //x
'
,
'
+-*/%&|^~!=<>(){}[]
\'
"
\\
/?;,
'
,
'
\t
'
);
test
(
'
advanceToEOS
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
var foo =bar("foo"); //x
'
);
assert
.
equal
(
lineStream
.
pos
(),
0
);
assert
.
equal
(
lineStream
.
peekToken
(),
'
var
'
);
check
(
lineStream
,
0
,
'
var
'
);
check
(
lineStream
,
5
,
'
foo
'
);
check
(
lineStream
,
9
,
'
=
'
);
check
(
lineStream
,
11
,
'
bar
'
);
check
(
lineStream
,
14
,
'
(
'
);
check
(
lineStream
,
15
,
'
"
'
);
check
(
lineStream
,
16
,
'
foo
'
);
check
(
lineStream
,
19
,
'
"
'
);
check
(
lineStream
,
20
,
'
)
'
);
check
(
lineStream
,
21
,
'
;
'
);
check
(
lineStream
,
22
,
'
/
'
);
check
(
lineStream
,
24
,
'
/
'
);
check
(
lineStream
,
25
,
'
x
'
);
checkPos
(
lineStream
,
26
);
lineStream
.
skipWhitespace
();
assert
.
ok
(
lineStream
.
eos
(),
'
Stream finished
'
);
});
test
(
'
javascript strings
'
,
()
=>
{
var
lineStream
=
newTokenStream
(
'
x = " my
\\
"string
\\
" ";
'
,
'
=()
\\
";/
'
,
'
\t
'
);
check
(
lineStream
,
0
,
'
x
'
);
check
(
lineStream
,
1
,
'
=
'
);
check
(
lineStream
,
3
,
'
"
'
);
check
(
lineStream
,
5
,
'
my
'
);
check
(
lineStream
,
9
,
'
\\
'
);
check
(
lineStream
,
11
,
'
"
'
);
check
(
lineStream
,
12
,
'
string
'
);
check
(
lineStream
,
18
,
'
\\
'
);
check
(
lineStream
,
19
,
'
"
'
);
check
(
lineStream
,
20
,
'
"
'
);
check
(
lineStream
,
22
,
'
;
'
);
lineStream
.
advanceToEOS
();
assert
.
ok
(
lineStream
.
eos
(),
'
Stream finished
'
);
});
test
(
'
peek
'
,
()
=>
{
var
lineStream
=
new
TokenStream
(
'
albert, bart, charlie, damon, erich
'
,
'
,
'
,
'
'
);
var
lineStream
=
new
LineStream
(
'
albert, bart, charlie, damon, erich
'
);
assert
.
equal
(
lineStream
.
peekToken
(),
'
albert
'
);
assert
.
equal
(
lineStream
.
peek
(),
'
a
'
);
lineStream
.
advance
(
1
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
albert
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
,
'
);
assert
.
equal
(
lineStream
.
peekToken
(),
'
bart
'
);
assert
.
equal
(
lineStream
.
peek
(),
'
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
bart
'
);
assert
.
equal
(
lineStream
.
peekToken
(),
'
,
'
);
assert
.
equal
(
lineStream
.
peek
(),
'
,
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
,
'
);
lineStream
.
advanceToEOS
();
assert
.
throws
(()
=>
{
lineStream
.
peekToken
();
});
assert
.
throws
(()
=>
{
lineStream
.
peek
();
});
});
test
(
'
next
'
,
()
=>
{
var
lineStream
=
newTokenStream
(
'
albert, bart, charlie, damon, erich
'
,
'
,
'
,
'
'
);
assert
.
equal
(
lineStream
.
peekToken
(),
'
albert
'
);
assert
.
equal
(
lineStream
.
next
(),
'
a
'
);
assert
.
equal
(
lineStream
.
next
(),
'
l
'
);
assert
.
equal
(
lineStream
.
next
(),
'
b
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
ert
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
,
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
bart
'
);
assert
.
equal
(
lineStream
.
peekToken
(),
'
,
'
);
assert
.
equal
(
lineStream
.
next
(),
'
,
'
);
assert
.
equal
(
lineStream
.
next
(),
'
'
);
assert
.
equal
(
lineStream
.
next
(),
'
c
'
);
assert
.
equal
(
lineStream
.
next
(),
'
h
'
);
assert
.
equal
(
lineStream
.
next
(),
'
a
'
);
assert
.
equal
(
lineStream
.
next
(),
'
r
'
);
assert
.
equal
(
lineStream
.
next
(),
'
l
'
);
assert
.
equal
(
lineStream
.
next
(),
'
i
'
);
assert
.
equal
(
lineStream
.
next
(),
'
e
'
);
assert
.
equal
(
lineStream
.
next
(),
'
,
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
damon
'
);
assert
.
equal
(
lineStream
.
peek
(),
'
l
'
);
lineStream
.
advanceToEOS
();
assert
.
throws
(()
=>
{
lineStream
.
peekToken
();
});
assert
.
throws
(()
=>
{
lineStream
.
peek
();
});
});
test
(
'
next & goBack
'
,
()
=>
{
var
lineStream
=
new
LineStream
(
'
albert, bart, charlie, damon, erich
'
);
lineStream
.
setTokenRules
(
'
,
'
,
'
'
);
assert
.
equal
(
lineStream
.
peekToken
(),
'
albert
'
);
assert
.
equal
(
lineStream
.
next
(),
'
a
'
);
assert
.
equal
(
lineStream
.
next
(),
'
l
'
);
assert
.
equal
(
lineStream
.
next
(),
'
b
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
ert
'
);
lineStream
.
goBack
(
6
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
albert
'
);
assert
.
equal
(
lineStream
.
next
(),
'
,
'
);
lineStream
.
goBack
(
7
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
albert
'
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
,
'
);
assert
.
equal
(
lineStream
.
next
(),
'
'
);
assert
.
equal
(
lineStream
.
next
(),
'
b
'
);
assert
.
equal
(
lineStream
.
next
(),
'
a
'
);
lineStream
.
goBack
(
3
);
assert
.
equal
(
lineStream
.
nextToken
(),
'
bart
'
);
lineStream
.
goBack
(
5
);
assert
.
equal
(
lineStream
.
next
(),
'
'
);
lineStream
.
advanceToEOS
();
assert
.
throws
(()
=>
{
lineStream
.
peekToken
();
});
assert
.
throws
(()
=>
{
lineStream
.
peek
();
});
});
});
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
浏览文件 @
9e05f824
...
...
@@ -69,7 +69,9 @@ class State extends AbstractState {
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
return
{
type
:
stream
.
next
()
===
'
.
'
?
''
:
'
text
'
};
let
chr
=
stream
.
peek
();
stream
.
advance
(
1
);
return
{
type
:
chr
===
'
.
'
?
''
:
'
text
'
};
}
}
...
...
src/vs/editor/test/common/modes/tokenization.test.ts
浏览文件 @
9e05f824
...
...
@@ -38,13 +38,27 @@ export class StateMemorizingLastWord extends AbstractState {
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
stream
.
setTokenRules
(
'
[]{}()==--
'
,
'
\t
\
u00a0
'
);
if
(
stream
.
skipWhitespace
()
!==
''
)
{
return
{
type
:
''
};
let
contents
=
stream
.
advanceToEOS
();
stream
.
goBack
(
contents
.
length
);
let
m
=
contents
.
match
(
/^
([\t
\u
00a0
]
+
)
/
);
if
(
m
)
{
stream
.
advance
(
m
[
0
].
length
);
return
{
type
:
''
};
}
var
word
=
stream
.
nextToken
();
m
=
contents
.
match
(
/^
([\[\]\{\}\(\)])
/
);
let
word
:
string
;
if
(
m
)
{
stream
.
advance
(
m
[
0
].
length
);
word
=
m
[
1
];
}
else
{
m
=
contents
.
match
(
/
([
a-zA-Z
]
+
)
/
);
stream
.
advance
(
m
[
0
].
length
);
word
=
m
[
1
];
}
return
{
type
:
this
.
getModeId
()
+
'
.
'
+
word
,
nextState
:
new
StateMemorizingLastWord
(
this
.
getModeId
(),
this
.
descriptor
,
word
)
...
...
@@ -169,7 +183,9 @@ suite('Editor Modes - Tokenization', () => {
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
return
{
type
:
stream
.
next
()
===
'
.
'
?
''
:
'
text
'
};
let
chr
=
stream
.
peek
();
stream
.
advance
(
1
);
return
{
type
:
chr
===
'
.
'
?
''
:
'
text
'
};
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录