Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
掘金者说
vscode
提交
ab02882f
V
vscode
项目概览
掘金者说
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ab02882f
编写于
7月 03, 2019
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Extract TokensStore
上级
cb58cf91
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
128 addition
and
79 deletion
+128
-79
src/vs/editor/common/model/textModel.ts
src/vs/editor/common/model/textModel.ts
+4
-4
src/vs/editor/common/model/textModelTokens.ts
src/vs/editor/common/model/textModelTokens.ts
+123
-74
src/vs/editor/test/common/model/model.modes.test.ts
src/vs/editor/test/common/model/model.modes.test.ts
+1
-1
未找到文件。
src/vs/editor/common/model/textModel.ts
浏览文件 @
ab02882f
...
...
@@ -1784,12 +1784,12 @@ export class TextModel extends Disposable implements model.ITextModel {
startLineNumber
=
Math
.
max
(
1
,
startLineNumber
);
endLineNumber
=
Math
.
min
(
this
.
getLineCount
(),
endLineNumber
);
if
(
endLineNumber
<=
this
.
_tokens
.
in
V
alidLineStartIndex
)
{
if
(
endLineNumber
<=
this
.
_tokens
.
in
v
alidLineStartIndex
)
{
// nothing to do
return
;
}
if
(
startLineNumber
<=
this
.
_tokens
.
in
V
alidLineStartIndex
)
{
if
(
startLineNumber
<=
this
.
_tokens
.
in
v
alidLineStartIndex
)
{
// tokenization has reached the viewport start...
this
.
forceTokenization
(
endLineNumber
);
return
;
...
...
@@ -1838,7 +1838,7 @@ export class TextModel extends Disposable implements model.ITextModel {
// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
this
.
_tokens
.
_
setIsInvalid
(
i
-
1
,
true
);
this
.
_tokens
.
_
invalidateLine
(
i
-
1
);
this
.
_tokens
.
_setState
(
i
-
1
,
state
);
state
=
r
.
endState
.
clone
();
eventBuilder
.
registerChangedTokens
(
i
);
...
...
@@ -1884,7 +1884,7 @@ export class TextModel extends Disposable implements model.ITextModel {
return
false
;
}
if
(
lineNumber
<
this
.
_tokens
.
in
V
alidLineStartIndex
+
1
)
{
if
(
lineNumber
<
this
.
_tokens
.
in
v
alidLineStartIndex
+
1
)
{
return
true
;
}
...
...
src/vs/editor/common/model/textModelTokens.ts
浏览文件 @
ab02882f
...
...
@@ -165,37 +165,22 @@ class ModelLineTokens {
}
}
export
class
ModelLinesTokens
{
public
readonly
languageIdentifier
:
LanguageIdentifier
;
public
readonly
tokenizationSupport
:
ITokenizationSupport
|
null
;
class
TokensStore
{
private
_tokens
:
ModelLineTokens
[];
private
_invalidLineStartIndex
:
number
;
private
_lastState
:
IState
|
null
;
_invalidLineStartIndex
:
number
;
_lastState
:
IState
|
null
;
constructor
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
|
null
)
{
this
.
languageIdentifier
=
languageIdentifier
;
this
.
tokenizationSupport
=
tokenizationSupport
;
constructor
(
initialState
:
IState
|
null
)
{
this
.
_tokens
=
[];
if
(
this
.
tokenizationSupport
)
{
let
initialState
:
IState
|
null
=
null
;
try
{
initialState
=
this
.
tokenizationSupport
.
getInitialState
();
}
catch
(
e
)
{
onUnexpectedError
(
e
);
this
.
tokenizationSupport
=
null
;
}
if
(
initialState
)
{
this
.
_tokens
[
0
]
=
new
ModelLineTokens
(
initialState
);
}
}
this
.
_invalidLineStartIndex
=
0
;
this
.
_lastState
=
null
;
if
(
initialState
)
{
this
.
_tokens
[
0
]
=
new
ModelLineTokens
(
initialState
);
}
}
public
get
in
V
alidLineStartIndex
()
{
public
get
in
v
alidLineStartIndex
()
{
return
this
.
_invalidLineStartIndex
;
}
...
...
@@ -215,15 +200,6 @@ export class ModelLinesTokens {
return
new
LineTokens
(
lineTokens
,
lineText
);
}
public
isCheapToTokenize
(
lineNumber
:
number
):
boolean
{
const
firstInvalidLineNumber
=
this
.
_invalidLineStartIndex
+
1
;
return
(
firstInvalidLineNumber
>=
lineNumber
);
}
public
hasLinesToTokenize
(
buffer
:
ITextBuffer
):
boolean
{
return
(
this
.
_invalidLineStartIndex
<
buffer
.
getLineCount
());
}
public
invalidateLine
(
lineIndex
:
number
):
void
{
this
.
_setIsInvalid
(
lineIndex
,
true
);
if
(
lineIndex
<
this
.
_invalidLineStartIndex
)
{
...
...
@@ -232,27 +208,27 @@ export class ModelLinesTokens {
}
}
_setIsInvalid
(
lineIndex
:
number
,
invalid
:
boolean
):
void
{
private
_setIsInvalid
(
lineIndex
:
number
,
invalid
:
boolean
):
void
{
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
])
{
this
.
_tokens
[
lineIndex
].
_invalid
=
invalid
;
}
}
_
isInvalid
(
lineIndex
:
number
):
boolean
{
public
isInvalid
(
lineIndex
:
number
):
boolean
{
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
])
{
return
this
.
_tokens
[
lineIndex
].
_invalid
;
}
return
true
;
}
_
getState
(
lineIndex
:
number
):
IState
|
null
{
public
getState
(
lineIndex
:
number
):
IState
|
null
{
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
])
{
return
this
.
_tokens
[
lineIndex
].
_state
;
}
return
null
;
}
_
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
public
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
let
target
:
ModelLineTokens
;
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
])
{
target
=
this
.
_tokens
[
lineIndex
];
...
...
@@ -284,7 +260,43 @@ export class ModelLinesTokens {
target
.
_lineTokens
=
tokens
.
buffer
;
}
_setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
public
setGoodTokens
(
topLevelLanguageId
:
LanguageId
,
linesLength
:
number
,
lineIndex
:
number
,
text
:
string
,
r
:
TokenizationResult2
):
void
{
const
endStateIndex
=
lineIndex
+
1
;
this
.
setTokens
(
topLevelLanguageId
,
lineIndex
,
text
.
length
,
r
.
tokens
);
this
.
_setIsInvalid
(
lineIndex
,
false
);
if
(
endStateIndex
<
linesLength
)
{
const
previousEndState
=
this
.
getState
(
endStateIndex
);
if
(
previousEndState
!==
null
&&
r
.
endState
.
equals
(
previousEndState
))
{
// The end state of this line remains the same
let
nextInvalidLineIndex
=
lineIndex
+
1
;
while
(
nextInvalidLineIndex
<
linesLength
)
{
if
(
this
.
isInvalid
(
nextInvalidLineIndex
))
{
break
;
}
if
(
nextInvalidLineIndex
+
1
<
linesLength
)
{
if
(
this
.
getState
(
nextInvalidLineIndex
+
1
)
===
null
)
{
break
;
}
}
else
{
if
(
this
.
_lastState
===
null
)
{
break
;
}
}
nextInvalidLineIndex
++
;
}
this
.
_invalidLineStartIndex
=
nextInvalidLineIndex
;
}
else
{
this
.
_invalidLineStartIndex
=
lineIndex
+
1
;
this
.
setState
(
endStateIndex
,
r
.
endState
);
}
}
else
{
this
.
_lastState
=
r
.
endState
;
this
.
_invalidLineStartIndex
=
linesLength
;
}
}
public
setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
])
{
this
.
_tokens
[
lineIndex
].
_state
=
state
;
}
else
{
...
...
@@ -374,6 +386,74 @@ export class ModelLinesTokens {
}
//#endregion
}
export
class
ModelLinesTokens
{
public
readonly
languageIdentifier
:
LanguageIdentifier
;
public
readonly
tokenizationSupport
:
ITokenizationSupport
|
null
;
public
readonly
store
:
TokensStore
;
constructor
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
|
null
)
{
this
.
languageIdentifier
=
languageIdentifier
;
this
.
tokenizationSupport
=
tokenizationSupport
;
let
initialState
:
IState
|
null
=
null
;
if
(
this
.
tokenizationSupport
)
{
try
{
initialState
=
this
.
tokenizationSupport
.
getInitialState
();
}
catch
(
e
)
{
onUnexpectedError
(
e
);
this
.
tokenizationSupport
=
null
;
}
}
this
.
store
=
new
TokensStore
(
initialState
);
}
public
get
invalidLineStartIndex
()
{
return
this
.
store
.
invalidLineStartIndex
;
}
public
getTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineText
:
string
):
LineTokens
{
return
this
.
store
.
getTokens
(
topLevelLanguageId
,
lineIndex
,
lineText
);
}
public
isCheapToTokenize
(
lineNumber
:
number
):
boolean
{
const
firstInvalidLineNumber
=
this
.
store
.
invalidLineStartIndex
+
1
;
return
(
firstInvalidLineNumber
>=
lineNumber
);
}
public
hasLinesToTokenize
(
buffer
:
ITextBuffer
):
boolean
{
return
(
this
.
store
.
invalidLineStartIndex
<
buffer
.
getLineCount
());
}
_isInvalid
(
lineIndex
:
number
):
boolean
{
return
this
.
store
.
isInvalid
(
lineIndex
);
}
_getState
(
lineIndex
:
number
):
IState
|
null
{
return
this
.
store
.
getState
(
lineIndex
);
}
_setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
this
.
store
.
setTokens
(
topLevelLanguageId
,
lineIndex
,
lineTextLength
,
tokens
);
}
_setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
this
.
store
.
setState
(
lineIndex
,
state
);
}
//#region Editing
public
applyEdits
(
range
:
Range
,
eolCount
:
number
,
firstLineLength
:
number
):
void
{
this
.
store
.
applyEdits
(
range
,
eolCount
,
firstLineLength
);
}
_invalidateLine
(
lineIndex
:
number
):
void
{
this
.
store
.
invalidateLine
(
lineIndex
);
}
//#endregion
//#region Tokenization
...
...
@@ -381,7 +461,7 @@ export class ModelLinesTokens {
if
(
!
this
.
hasLinesToTokenize
(
buffer
))
{
return
buffer
.
getLineCount
()
+
1
;
}
const
lineNumber
=
this
.
_
invalidLineStartIndex
+
1
;
const
lineNumber
=
this
.
store
.
invalidLineStartIndex
+
1
;
this
.
_updateTokensUntilLine
(
buffer
,
eventBuilder
,
lineNumber
);
return
lineNumber
;
}
...
...
@@ -405,7 +485,7 @@ export class ModelLinesTokens {
public
_updateTokensUntilLine
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
lineNumber
:
number
):
void
{
if
(
!
this
.
tokenizationSupport
)
{
this
.
_invalidLineStartIndex
=
buffer
.
getLineCount
();
this
.
store
.
_invalidLineStartIndex
=
buffer
.
getLineCount
();
return
;
}
...
...
@@ -413,8 +493,7 @@ export class ModelLinesTokens {
const
endLineIndex
=
lineNumber
-
1
;
// Validate all states up to and including endLineIndex
for
(
let
lineIndex
=
this
.
_invalidLineStartIndex
;
lineIndex
<=
endLineIndex
;
lineIndex
++
)
{
const
endStateIndex
=
lineIndex
+
1
;
for
(
let
lineIndex
=
this
.
store
.
invalidLineStartIndex
;
lineIndex
<=
endLineIndex
;
lineIndex
++
)
{
const
text
=
buffer
.
getLineContent
(
lineIndex
+
1
);
const
lineStartState
=
this
.
_getState
(
lineIndex
);
...
...
@@ -431,40 +510,10 @@ export class ModelLinesTokens {
if
(
!
r
)
{
r
=
nullTokenize2
(
this
.
languageIdentifier
.
id
,
text
,
lineStartState
,
0
);
}
this
.
_setTokens
(
this
.
languageIdentifier
.
id
,
lineIndex
,
text
.
length
,
r
.
tokens
);
this
.
store
.
setGoodTokens
(
this
.
languageIdentifier
.
id
,
linesLength
,
lineIndex
,
text
,
r
);
eventBuilder
.
registerChangedTokens
(
lineIndex
+
1
);
this
.
_setIsInvalid
(
lineIndex
,
false
);
if
(
endStateIndex
<
linesLength
)
{
const
previousEndState
=
this
.
_getState
(
endStateIndex
);
if
(
previousEndState
!==
null
&&
r
.
endState
.
equals
(
previousEndState
))
{
// The end state of this line remains the same
let
nextInvalidLineIndex
=
lineIndex
+
1
;
while
(
nextInvalidLineIndex
<
linesLength
)
{
if
(
this
.
_isInvalid
(
nextInvalidLineIndex
))
{
break
;
}
if
(
nextInvalidLineIndex
+
1
<
linesLength
)
{
if
(
this
.
_getState
(
nextInvalidLineIndex
+
1
)
===
null
)
{
break
;
}
}
else
{
if
(
this
.
_lastState
===
null
)
{
break
;
}
}
nextInvalidLineIndex
++
;
}
this
.
_invalidLineStartIndex
=
Math
.
max
(
this
.
_invalidLineStartIndex
,
nextInvalidLineIndex
);
lineIndex
=
nextInvalidLineIndex
-
1
;
// -1 because the outer loop increments it
}
else
{
this
.
_setState
(
endStateIndex
,
r
.
endState
);
}
}
else
{
this
.
_lastState
=
r
.
endState
;
}
lineIndex
=
this
.
store
.
invalidLineStartIndex
-
1
;
// -1 because the outer loop increments it
}
this
.
_invalidLineStartIndex
=
Math
.
max
(
this
.
_invalidLineStartIndex
,
endLineIndex
+
1
);
}
// #endregion
...
...
src/vs/editor/test/common/model/model.modes.test.ts
浏览文件 @
ab02882f
...
...
@@ -198,7 +198,7 @@ suite('Editor Model - Model Modes 2', () => {
for
(
i
=
0
;
i
<
len
;
i
++
)
{
stateEqual
(
model
.
_tokens
.
_getState
(
i
)
!
,
states
[
i
]);
}
stateEqual
(
(
<
any
>
model
).
_tokens
.
_lastState
,
states
[
len
]);
stateEqual
(
model
.
_tokens
.
store
.
_lastState
!
,
states
[
len
]);
}
let
thisModel
:
TextModel
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录