Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
掘金者说
vscode
提交
5598ee09
V
vscode
项目概览
掘金者说
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
5598ee09
编写于
2月 22, 2016
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fixes #2968: Do not massage TM tokens & merge sequential identical ones
上级
5c5752c7
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
7 addition
and
80 deletion
+7
-80
src/vs/editor/node/textMate/TMSyntax.ts
src/vs/editor/node/textMate/TMSyntax.ts
+7
-80
未找到文件。
src/vs/editor/node/textMate/TMSyntax.ts
浏览文件 @
5598ee09
...
@@ -5,7 +5,6 @@
...
@@ -5,7 +5,6 @@
'
use strict
'
;
'
use strict
'
;
import
*
as
nls
from
'
vs/nls
'
;
import
*
as
nls
from
'
vs/nls
'
;
import
*
as
collections
from
'
vs/base/common/collections
'
;
import
{
onUnexpectedError
}
from
'
vs/base/common/errors
'
;
import
{
onUnexpectedError
}
from
'
vs/base/common/errors
'
;
import
*
as
paths
from
'
vs/base/common/paths
'
;
import
*
as
paths
from
'
vs/base/common/paths
'
;
import
{
IMessageCollector
,
PluginsRegistry
}
from
'
vs/platform/plugins/common/pluginsRegistry
'
;
import
{
IMessageCollector
,
PluginsRegistry
}
from
'
vs/platform/plugins/common/pluginsRegistry
'
;
...
@@ -159,64 +158,16 @@ class Tokenizer {
...
@@ -159,64 +158,16 @@ class Tokenizer {
modeTransitions
:
[{
startIndex
:
offsetDelta
,
mode
:
freshState
.
getMode
()
}],
modeTransitions
:
[{
startIndex
:
offsetDelta
,
mode
:
freshState
.
getMode
()
}],
};
};
let
lastTokenType
:
string
=
null
;
for
(
let
tokenIndex
=
0
,
len
=
textMateResult
.
tokens
.
length
;
tokenIndex
<
len
;
tokenIndex
++
)
{
for
(
let
tokenIndex
=
0
,
len
=
textMateResult
.
tokens
.
length
;
tokenIndex
<
len
;
tokenIndex
++
)
{
let
token
=
textMateResult
.
tokens
[
tokenIndex
];
let
token
=
textMateResult
.
tokens
[
tokenIndex
];
let
tokenStartIndex
=
token
.
startIndex
;
let
tokenStartIndex
=
token
.
startIndex
;
let
tokenEndIndex
=
(
tokenIndex
+
1
<
len
?
textMateResult
.
tokens
[
tokenIndex
+
1
].
startIndex
:
line
.
length
);
let
t
=
decodeTextMateToken
(
this
.
_modeId
,
token
);
let
t
=
decodeTextMateToken
(
this
.
_modeId
,
token
);
if
(
t
.
isOpaqueToken
)
{
// do not push a new token if the type is exactly the same (also helps with ligatures)
// Should not do any smartness to detect brackets on this token
if
(
t
.
tokenType
!==
lastTokenType
)
{
ret
.
tokens
.
push
(
new
Token
(
tokenStartIndex
+
offsetDelta
,
t
.
tokenType
));
continue
;
}
let
i
:
number
,
charCode
:
number
,
isBracket
:
string
;
for
(
i
=
tokenStartIndex
;
i
<
tokenEndIndex
;
i
++
)
{
charCode
=
line
.
charCodeAt
(
i
);
isBracket
=
null
;
switch
(
charCode
)
{
case
_openParen
:
// (
isBracket
=
'
delimiter.paren
'
;
break
;
case
_closeParen
:
// )
isBracket
=
'
delimiter.paren
'
;
break
;
case
_openCurly
:
// {
isBracket
=
'
delimiter.curly
'
;
break
;
case
_closeCurly
:
// }
isBracket
=
'
delimiter.curly
'
;
break
;
case
_openSquare
:
// [
isBracket
=
'
delimiter.square
'
;
break
;
case
_closeSquare
:
// ]
isBracket
=
'
delimiter.square
'
;
break
;
}
if
(
isBracket
)
{
if
(
tokenStartIndex
<
i
)
{
// push a token before character `i`
ret
.
tokens
.
push
(
new
Token
(
tokenStartIndex
+
offsetDelta
,
t
.
tokenType
));
tokenStartIndex
=
i
;
}
// push character `i` as a token
ret
.
tokens
.
push
(
new
Token
(
tokenStartIndex
+
offsetDelta
,
isBracket
+
'
.
'
+
t
.
modeToken
));
tokenStartIndex
=
i
+
1
;
}
}
if
(
tokenStartIndex
<
tokenEndIndex
)
{
// push the remaining text as a token
ret
.
tokens
.
push
(
new
Token
(
tokenStartIndex
+
offsetDelta
,
t
.
tokenType
));
ret
.
tokens
.
push
(
new
Token
(
tokenStartIndex
+
offsetDelta
,
t
.
tokenType
));
lastTokenType
=
t
.
tokenType
;
}
}
}
}
...
@@ -237,10 +188,9 @@ function decodeTextMateToken(modeId:string, entry: ITMToken) {
...
@@ -237,10 +188,9 @@ function decodeTextMateToken(modeId:string, entry: ITMToken) {
}
}
}
}
let
tokenTypes
:
string
[]
=
[];
let
tokenTypes
:
string
[]
=
[];
let
isOpaqueToken
=
dedupTokens
(
tokenTypeArray
,
modeToken
,
tokenTypes
);
dedupTokens
(
tokenTypeArray
,
modeToken
,
tokenTypes
);
return
{
return
{
isOpaqueToken
:
isOpaqueToken
,
tokenType
:
tokenTypes
.
join
(
'
.
'
),
tokenType
:
tokenTypes
.
join
(
'
.
'
),
modeToken
:
modeId
modeToken
:
modeId
};
};
...
@@ -250,13 +200,12 @@ function decodeTextMateToken(modeId:string, entry: ITMToken) {
...
@@ -250,13 +200,12 @@ function decodeTextMateToken(modeId:string, entry: ITMToken) {
* Remove duplicate entries, collect result in `result`, place `modeToken` at the end
* Remove duplicate entries, collect result in `result`, place `modeToken` at the end
* and detect if this is a comment => return true if it looks like a comment
* and detect if this is a comment => return true if it looks like a comment
*/
*/
function
dedupTokens
(
tokenTypeArray
:
string
[],
modeToken
:
string
,
result
:
string
[]):
boolean
{
function
dedupTokens
(
tokenTypeArray
:
string
[],
modeToken
:
string
,
result
:
string
[]):
void
{
tokenTypeArray
.
sort
();
tokenTypeArray
.
sort
();
var
prev
:
string
=
null
,
var
prev
:
string
=
null
,
curr
:
string
=
null
,
curr
:
string
=
null
;
isOpaqueToken
=
false
;
for
(
var
i
=
0
,
len
=
tokenTypeArray
.
length
;
i
<
len
;
i
++
)
{
for
(
var
i
=
0
,
len
=
tokenTypeArray
.
length
;
i
<
len
;
i
++
)
{
prev
=
curr
;
prev
=
curr
;
...
@@ -267,29 +216,7 @@ function dedupTokens(tokenTypeArray:string[], modeToken:string, result:string[])
...
@@ -267,29 +216,7 @@ function dedupTokens(tokenTypeArray:string[], modeToken:string, result:string[])
}
}
result
.
push
(
curr
);
result
.
push
(
curr
);
if
(
!
isOpaqueToken
&&
(
curr
===
'
comment
'
||
curr
===
'
string
'
||
curr
===
'
regexp
'
))
{
isOpaqueToken
=
true
;
}
}
}
result
.
push
(
modeToken
);
result
.
push
(
modeToken
);
return
isOpaqueToken
;
}
}
var
_openParen
=
'
(
'
.
charCodeAt
(
0
);
var
_closeParen
=
'
)
'
.
charCodeAt
(
0
);
var
_openCurly
=
'
{
'
.
charCodeAt
(
0
);
var
_closeCurly
=
'
}
'
.
charCodeAt
(
0
);
var
_openSquare
=
'
[
'
.
charCodeAt
(
0
);
var
_closeSquare
=
'
]
'
.
charCodeAt
(
0
);
var
characterToBracket
=
collections
.
createNumberDictionary
<
number
>
();
characterToBracket
[
'
(
'
.
charCodeAt
(
0
)]
=
Bracket
.
Open
;
characterToBracket
[
'
)
'
.
charCodeAt
(
0
)]
=
Bracket
.
Close
;
characterToBracket
[
'
{
'
.
charCodeAt
(
0
)]
=
Bracket
.
Open
;
characterToBracket
[
'
}
'
.
charCodeAt
(
0
)]
=
Bracket
.
Close
;
characterToBracket
[
'
[
'
.
charCodeAt
(
0
)]
=
Bracket
.
Open
;
characterToBracket
[
'
]
'
.
charCodeAt
(
0
)]
=
Bracket
.
Close
;
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录