Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
xxadev
vscode
提交
8963c399
V
vscode
项目概览
xxadev
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
8963c399
编写于
9月 12, 2016
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Modernize and adopt CharacterClassifier in LineStream
上级
eb4d355a
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
124 addition
and
122 deletion
+124
-122
src/vs/editor/common/modes/lineStream.ts
src/vs/editor/common/modes/lineStream.ts
+124
-122
未找到文件。
src/vs/editor/common/modes/lineStream.ts
浏览文件 @
8963c399
...
...
@@ -5,53 +5,55 @@
'
use strict
'
;
import
{
IStream
}
from
'
vs/editor/common/modes
'
;
import
{
CharacterClassifier
}
from
'
vs/editor/common/core/characterClassifier
'
;
export
class
LineStream
implements
IStream
{
class
CharacterSet
{
static
STRING_TO_ARRAY_CACHE
:{
[
key
:
string
]:
boolean
[];
}
=
{};
private
static
_CACHE
:{
[
key
:
string
]:
CharacterSet
;
}
=
{};
// TODO@Alex unbounded cache
/*protected*/
_source
:
string
;
private
sourceLength
:
number
;
/*protected*/
_pos
:
number
;
private
whitespace
:
string
;
private
whitespaceArr
:
boolean
[];
private
separators
:
string
;
private
separatorsArr
:
boolean
[];
private
tokenStart
:
number
;
private
tokenEnd
:
number
;
public
static
getOrCreate
(
source
:
string
):
CharacterSet
{
if
(
!
CharacterSet
.
_CACHE
.
hasOwnProperty
(
source
))
{
CharacterSet
.
_CACHE
[
source
]
=
new
CharacterSet
(
source
);
}
return
CharacterSet
.
_CACHE
[
source
];
}
private
_classifier
:
CharacterClassifier
<
boolean
>
;
constructor
(
source
:
string
)
{
this
.
_source
=
source
;
this
.
sourceLength
=
source
.
length
;
this
.
_pos
=
0
;
this
.
whitespace
=
'
\t
\
u00a0
'
;
this
.
whitespaceArr
=
this
.
stringToArray
(
this
.
whitespace
);
this
.
separators
=
''
;
this
.
separatorsArr
=
this
.
stringToArray
(
this
.
separators
);
this
.
tokenStart
=
-
1
;
this
.
tokenEnd
=
-
1
;
this
.
_classifier
=
new
CharacterClassifier
<
boolean
>
(
false
);
for
(
let
i
=
0
,
len
=
source
.
length
;
i
<
len
;
i
++
)
{
this
.
_classifier
.
set
(
source
.
charCodeAt
(
i
),
true
);
}
}
private
stringToArray
(
str
:
string
):
boolean
[]
{
if
(
!
LineStream
.
STRING_TO_ARRAY_CACHE
.
hasOwnProperty
(
str
))
{
LineStream
.
STRING_TO_ARRAY_CACHE
[
str
]
=
this
.
actualStringToArray
(
str
);
}
return
LineStream
.
STRING_TO_ARRAY_CACHE
[
str
];
public
contains
(
charCode
:
number
):
boolean
{
return
this
.
_classifier
.
get
(
charCode
);
}
}
private
actualStringToArray
(
str
:
string
):
boolean
[]
{
let
maxCharCode
=
0
;
for
(
let
i
=
0
;
i
<
str
.
length
;
i
++
)
{
maxCharCode
=
Math
.
max
(
maxCharCode
,
str
.
charCodeAt
(
i
));
}
let
r
:
boolean
[]
=
[];
for
(
let
i
=
0
;
i
<=
maxCharCode
;
i
++
)
{
r
[
i
]
=
false
;
}
for
(
let
i
=
0
;
i
<
str
.
length
;
i
++
)
{
r
[
str
.
charCodeAt
(
i
)]
=
true
;
}
return
r
;
export
class
LineStream
implements
IStream
{
private
_source
:
string
;
private
_sourceLength
:
number
;
private
_pos
:
number
;
private
_whitespace
:
string
;
private
_whitespaceArr
:
CharacterSet
;
private
_separators
:
string
;
private
_separatorsArr
:
CharacterSet
;
private
_tokenStart
:
number
;
private
_tokenEnd
:
number
;
constructor
(
source
:
string
)
{
this
.
_source
=
source
;
this
.
_sourceLength
=
source
.
length
;
this
.
_pos
=
0
;
this
.
_whitespace
=
'
\t
\
u00a0
'
;
this
.
_whitespaceArr
=
CharacterSet
.
getOrCreate
(
this
.
_whitespace
);
this
.
_separators
=
''
;
this
.
_separatorsArr
=
CharacterSet
.
getOrCreate
(
this
.
_separators
);
this
.
_tokenStart
=
-
1
;
this
.
_tokenEnd
=
-
1
;
}
public
pos
():
number
{
...
...
@@ -59,12 +61,12 @@ export class LineStream implements IStream {
}
public
eos
()
{
return
this
.
_pos
>=
this
.
sourceLength
;
return
this
.
_pos
>=
this
.
_
sourceLength
;
}
public
peek
():
string
{
// Check EOS
if
(
this
.
_pos
>=
this
.
sourceLength
)
{
if
(
this
.
_pos
>=
this
.
_
sourceLength
)
{
throw
new
Error
(
'
Stream is at the end
'
);
}
return
this
.
_source
[
this
.
_pos
];
...
...
@@ -72,26 +74,26 @@ export class LineStream implements IStream {
public
next
():
string
{
// Check EOS
if
(
this
.
_pos
>=
this
.
sourceLength
)
{
if
(
this
.
_pos
>=
this
.
_
sourceLength
)
{
throw
new
Error
(
'
Stream is at the end
'
);
}
// Reset peeked token
this
.
tokenStart
=
-
1
;
this
.
tokenEnd
=
-
1
;
this
.
_
tokenStart
=
-
1
;
this
.
_
tokenEnd
=
-
1
;
return
this
.
_source
[
this
.
_pos
++
];
}
public
next2
():
void
{
// Check EOS
if
(
this
.
_pos
>=
this
.
sourceLength
)
{
if
(
this
.
_pos
>=
this
.
_
sourceLength
)
{
throw
new
Error
(
'
Stream is at the end
'
);
}
// Reset peeked token
this
.
tokenStart
=
-
1
;
this
.
tokenEnd
=
-
1
;
this
.
_
tokenStart
=
-
1
;
this
.
_
tokenEnd
=
-
1
;
this
.
_pos
++
;
}
...
...
@@ -100,11 +102,11 @@ export class LineStream implements IStream {
if
(
n
===
0
)
{
return
''
;
}
var
oldPos
=
this
.
_pos
;
const
oldPos
=
this
.
_pos
;
this
.
_pos
+=
n
;
// Reset peeked token
this
.
tokenStart
=
-
1
;
this
.
tokenEnd
=
-
1
;
this
.
_
tokenStart
=
-
1
;
this
.
_
tokenEnd
=
-
1
;
return
this
.
_source
.
substring
(
oldPos
,
this
.
_pos
);
}
...
...
@@ -114,14 +116,14 @@ export class LineStream implements IStream {
}
this
.
_pos
+=
n
;
// Reset peeked token
this
.
tokenStart
=
-
1
;
this
.
tokenEnd
=
-
1
;
this
.
_
tokenStart
=
-
1
;
this
.
_
tokenEnd
=
-
1
;
return
n
;
}
public
advanceToEOS
():
string
{
var
oldPos
=
this
.
_pos
;
this
.
_pos
=
this
.
sourceLength
;
const
oldPos
=
this
.
_pos
;
this
.
_pos
=
this
.
_
sourceLength
;
this
.
resetPeekedToken
();
return
this
.
_source
.
substring
(
oldPos
,
this
.
_pos
);
}
...
...
@@ -131,10 +133,10 @@ export class LineStream implements IStream {
this
.
resetPeekedToken
();
}
private
createPeeker
(
condition
:
any
):()
=>
number
{
private
createPeeker
(
condition
:
RegExp
|
string
):()
=>
number
{
if
(
condition
instanceof
RegExp
)
{
return
()
=>
{
var
result
=
condition
.
exec
(
this
.
_source
.
substr
(
this
.
_pos
));
let
result
=
condition
.
exec
(
this
.
_source
.
substr
(
this
.
_pos
));
if
(
result
===
null
)
{
return
0
;
}
else
if
(
result
.
index
!==
0
)
{
...
...
@@ -142,13 +144,14 @@ export class LineStream implements IStream {
}
return
result
[
0
].
length
;
};
}
else
if
((
condition
instanceof
String
||
(
typeof
condition
)
===
'
string
'
)
&&
condition
)
{
}
else
if
((
typeof
condition
===
'
string
'
)
&&
condition
)
{
return
()
=>
{
var
len
=
(
<
String
>
condition
).
length
,
match
=
this
.
_pos
+
len
<=
this
.
sourceLength
;
for
(
var
i
=
0
;
match
&&
i
<
len
;
i
++
)
{
match
=
this
.
_source
.
charCodeAt
(
this
.
_pos
+
i
)
===
(
<
String
>
condition
).
charCodeAt
(
i
);
const
len
=
condition
.
length
;
let
match
=
(
this
.
_pos
+
len
<=
this
.
_sourceLength
);
for
(
let
i
=
0
;
match
&&
i
<
len
;
i
++
)
{
match
=
this
.
_source
.
charCodeAt
(
this
.
_pos
+
i
)
===
condition
.
charCodeAt
(
i
);
}
return
match
?
len
:
0
;
return
(
match
?
len
:
0
)
;
};
}
throw
new
Error
(
'
Condition must be either a regular expression, function or a non-empty string
'
);
...
...
@@ -156,16 +159,15 @@ export class LineStream implements IStream {
// --- BEGIN `_advanceIfStringCaseInsensitive`
private
_advanceIfStringCaseInsensitive
(
condition
:
string
):
number
{
var
oldPos
=
this
.
_pos
,
source
=
this
.
_source
,
len
=
condition
.
length
,
i
:
number
;
const
oldPos
=
this
.
_pos
;
const
source
=
this
.
_source
;
const
len
=
condition
.
length
;
if
(
len
<
1
||
oldPos
+
len
>
this
.
sourceLength
)
{
if
(
len
<
1
||
oldPos
+
len
>
this
.
_
sourceLength
)
{
return
0
;
}
for
(
i
=
0
;
i
<
len
;
i
++
)
{
for
(
let
i
=
0
;
i
<
len
;
i
++
)
{
if
(
source
.
charAt
(
oldPos
+
i
).
toLowerCase
()
!==
condition
.
charAt
(
i
).
toLowerCase
())
{
return
0
;
}
...
...
@@ -183,16 +185,15 @@ export class LineStream implements IStream {
// --- BEGIN `advanceIfString`
private
_advanceIfString
(
condition
:
string
):
number
{
var
oldPos
=
this
.
_pos
,
source
=
this
.
_source
,
len
=
condition
.
length
,
i
:
number
;
const
oldPos
=
this
.
_pos
;
const
source
=
this
.
_source
;
const
len
=
condition
.
length
;
if
(
len
<
1
||
oldPos
+
len
>
this
.
sourceLength
)
{
if
(
len
<
1
||
oldPos
+
len
>
this
.
_
sourceLength
)
{
return
0
;
}
for
(
i
=
0
;
i
<
len
;
i
++
)
{
for
(
let
i
=
0
;
i
<
len
;
i
++
)
{
if
(
source
.
charCodeAt
(
oldPos
+
i
)
!==
condition
.
charCodeAt
(
i
))
{
return
0
;
}
...
...
@@ -210,7 +211,7 @@ export class LineStream implements IStream {
// --- BEGIN `advanceIfString`
private
_advanceIfCharCode
(
charCode
:
number
):
number
{
if
(
this
.
_pos
<
this
.
sourceLength
&&
this
.
_source
.
charCodeAt
(
this
.
_pos
)
===
charCode
)
{
if
(
this
.
_pos
<
this
.
_
sourceLength
&&
this
.
_source
.
charCodeAt
(
this
.
_pos
)
===
charCode
)
{
return
1
;
}
...
...
@@ -226,7 +227,7 @@ export class LineStream implements IStream {
// --- BEGIN `advanceIfRegExp`
private
_advanceIfRegExp
(
condition
:
RegExp
):
number
{
if
(
this
.
_pos
>=
this
.
sourceLength
)
{
if
(
this
.
_pos
>=
this
.
_
sourceLength
)
{
return
0
;
}
if
(
!
condition
.
test
(
this
.
_source
.
substr
(
this
.
_pos
)))
{
...
...
@@ -242,14 +243,14 @@ export class LineStream implements IStream {
}
// --- END
private
advanceLoop
(
condition
:
any
,
isWhile
:
boolean
,
including
:
boolean
):
string
{
private
advanceLoop
(
condition
:
RegExp
|
string
,
isWhile
:
boolean
,
including
:
boolean
):
string
{
if
(
this
.
eos
())
{
return
''
;
}
var
peeker
=
this
.
createPeeker
(
condition
);
var
oldPos
=
this
.
_pos
;
var
n
=
0
;
var
f
=
null
;
const
peeker
=
this
.
createPeeker
(
condition
);
const
oldPos
=
this
.
_pos
;
let
n
=
0
;
let
f
=
null
;
if
(
isWhile
)
{
f
=
(
n
)
=>
{
return
n
>
0
;
...
...
@@ -272,11 +273,11 @@ export class LineStream implements IStream {
return
this
.
_source
.
substring
(
oldPos
,
this
.
_pos
);
}
public
advanceWhile
(
condition
:
any
):
string
{
public
advanceWhile
(
condition
:
RegExp
|
string
):
string
{
return
this
.
advanceLoop
(
condition
,
true
,
false
);
}
public
advanceUntil
(
condition
:
any
,
including
:
boolean
):
string
{
public
advanceUntil
(
condition
:
RegExp
|
string
,
including
:
boolean
):
string
{
return
this
.
advanceLoop
(
condition
,
false
,
including
);
}
...
...
@@ -286,12 +287,12 @@ export class LineStream implements IStream {
return
0
;
}
var
oldPos
=
this
.
_pos
;
var
index
=
this
.
_source
.
indexOf
(
condition
,
oldPos
);
const
oldPos
=
this
.
_pos
;
const
index
=
this
.
_source
.
indexOf
(
condition
,
oldPos
);
if
(
index
===
-
1
)
{
// String was not found => advanced to `eos`
return
(
this
.
sourceLength
-
oldPos
);
return
(
this
.
_
sourceLength
-
oldPos
);
}
if
(
including
)
{
...
...
@@ -311,16 +312,16 @@ export class LineStream implements IStream {
// --- END
private
resetPeekedToken
()
{
this
.
tokenStart
=
-
1
;
this
.
tokenEnd
=
-
1
;
this
.
_
tokenStart
=
-
1
;
this
.
_
tokenEnd
=
-
1
;
}
public
setTokenRules
(
separators
:
string
,
whitespace
:
string
):
void
{
if
(
this
.
separators
!==
separators
||
this
.
whitespace
!==
whitespace
)
{
this
.
separators
=
separators
;
this
.
separatorsArr
=
this
.
stringToArray
(
this
.
separators
);
this
.
whitespace
=
whitespace
;
this
.
whitespaceArr
=
this
.
stringToArray
(
this
.
whitespace
);
if
(
this
.
_separators
!==
separators
||
this
.
_
whitespace
!==
whitespace
)
{
this
.
_
separators
=
separators
;
this
.
_separatorsArr
=
CharacterSet
.
getOrCreate
(
this
.
_
separators
);
this
.
_
whitespace
=
whitespace
;
this
.
_whitespaceArr
=
CharacterSet
.
getOrCreate
(
this
.
_
whitespace
);
this
.
resetPeekedToken
();
}
}
...
...
@@ -328,15 +329,16 @@ export class LineStream implements IStream {
// --- tokens
public
peekToken
():
string
{
if
(
this
.
tokenStart
!==
-
1
)
{
return
this
.
_source
.
substring
(
this
.
tokenStart
,
this
.
tokenEnd
);
if
(
this
.
_
tokenStart
!==
-
1
)
{
return
this
.
_source
.
substring
(
this
.
_tokenStart
,
this
.
_
tokenEnd
);
}
var
source
=
this
.
_source
,
sourceLength
=
this
.
sourceLength
,
whitespaceArr
=
this
.
whitespaceArr
,
separatorsArr
=
this
.
separatorsArr
,
tokenStart
=
this
.
_pos
;
const
source
=
this
.
_source
;
const
sourceLength
=
this
.
_sourceLength
;
const
whitespaceArr
=
this
.
_whitespaceArr
;
const
separatorsArr
=
this
.
_separatorsArr
;
let
tokenStart
=
this
.
_pos
;
// Check EOS
if
(
tokenStart
>=
sourceLength
)
{
...
...
@@ -344,48 +346,48 @@ export class LineStream implements IStream {
}
// Skip whitespace
while
(
whitespaceArr
[
source
.
charCodeAt
(
tokenStart
)]
&&
tokenStart
<
sourceLength
)
{
while
(
whitespaceArr
.
contains
(
source
.
charCodeAt
(
tokenStart
))
&&
tokenStart
<
sourceLength
)
{
tokenStart
++
;
}
var
tokenEnd
=
tokenStart
;
// If a separator is hit, it is a token
if
(
separatorsArr
[
source
.
charCodeAt
(
tokenEnd
)]
&&
tokenEnd
<
sourceLength
)
{
if
(
separatorsArr
.
contains
(
source
.
charCodeAt
(
tokenEnd
))
&&
tokenEnd
<
sourceLength
)
{
tokenEnd
++
;
}
else
{
// Advance until a separator or a whitespace is hit
while
(
!
separatorsArr
[
source
.
charCodeAt
(
tokenEnd
)]
&&
!
whitespaceArr
[
source
.
charCodeAt
(
tokenEnd
)]
&&
tokenEnd
<
sourceLength
)
{
while
(
!
separatorsArr
.
contains
(
source
.
charCodeAt
(
tokenEnd
))
&&
!
whitespaceArr
.
contains
(
source
.
charCodeAt
(
tokenEnd
))
&&
tokenEnd
<
sourceLength
)
{
tokenEnd
++
;
}
}
// Cache peeked token
this
.
tokenStart
=
tokenStart
;
this
.
tokenEnd
=
tokenEnd
;
this
.
_
tokenStart
=
tokenStart
;
this
.
_
tokenEnd
=
tokenEnd
;
return
source
.
substring
(
tokenStart
,
tokenEnd
);
}
public
nextToken
():
string
{
// Check EOS
if
(
this
.
_pos
>=
this
.
sourceLength
)
{
if
(
this
.
_pos
>=
this
.
_
sourceLength
)
{
throw
new
Error
(
'
Stream is at the end
'
);
}
// Peek token if necessary
var
result
:
string
;
if
(
this
.
tokenStart
===
-
1
)
{
let
result
:
string
;
if
(
this
.
_
tokenStart
===
-
1
)
{
result
=
this
.
peekToken
();
}
else
{
result
=
this
.
_source
.
substring
(
this
.
tokenStart
,
this
.
tokenEnd
);
result
=
this
.
_source
.
substring
(
this
.
_tokenStart
,
this
.
_
tokenEnd
);
}
// Advance to tokenEnd
this
.
_pos
=
this
.
tokenEnd
;
this
.
_pos
=
this
.
_
tokenEnd
;
// Reset peeked token
this
.
tokenStart
=
-
1
;
this
.
tokenEnd
=
-
1
;
this
.
_
tokenStart
=
-
1
;
this
.
_
tokenEnd
=
-
1
;
return
result
;
}
...
...
@@ -393,26 +395,26 @@ export class LineStream implements IStream {
// -- whitespace
public
peekWhitespace
():
string
{
var
source
=
this
.
_source
,
sourceLength
=
this
.
sourceLength
,
whitespaceArr
=
this
.
whitespaceArr
,
peek
=
this
.
_pos
;
const
source
=
this
.
_source
;
const
sourceLength
=
this
.
_sourceLength
;
const
whitespaceArr
=
this
.
_whitespaceArr
;
while
(
whitespaceArr
[
source
.
charCodeAt
(
peek
)]
&&
peek
<
sourceLength
)
{
let
peek
=
this
.
_pos
;
while
(
whitespaceArr
.
contains
(
source
.
charCodeAt
(
peek
))
&&
peek
<
sourceLength
)
{
peek
++
;
}
return
source
.
substring
(
this
.
_pos
,
peek
);
}
// --- BEGIN `
advanceIfRegExp
`
// --- BEGIN `
skipWhitespace
`
private
_skipWhitespace
():
number
{
var
source
=
this
.
_source
,
sourceLength
=
this
.
sourceLength
,
whitespaceArr
=
this
.
whitespaceArr
,
oldPos
=
this
.
_pos
,
peek
=
this
.
_pos
;
const
source
=
this
.
_source
;
const
sourceLength
=
this
.
_sourceLength
;
const
whitespaceArr
=
this
.
_whitespaceArr
;
const
oldPos
=
this
.
_pos
;
while
(
whitespaceArr
[
source
.
charCodeAt
(
peek
)]
&&
peek
<
sourceLength
)
{
let
peek
=
this
.
_pos
;
while
(
whitespaceArr
.
contains
(
source
.
charCodeAt
(
peek
))
&&
peek
<
sourceLength
)
{
peek
++
;
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录