Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
掘金者说
vscode
提交
7f833a91
V
vscode
项目概览
掘金者说
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7f833a91
编写于
7月 03, 2019
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
More cleanup in tokens code
上级
631e22aa
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
118 addition
and
118 deletion
+118
-118
src/vs/editor/common/model/textModelTokens.ts
src/vs/editor/common/model/textModelTokens.ts
+109
-115
src/vs/editor/test/common/model/model.line.test.ts
src/vs/editor/test/common/model/model.line.test.ts
+9
-3
未找到文件。
src/vs/editor/common/model/textModelTokens.ts
浏览文件 @
7f833a91
...
...
@@ -31,12 +31,12 @@ const enum Constants {
}
class
ModelLineTokens
{
public
s
tate
:
IState
|
null
;
public
beginS
tate
:
IState
|
null
;
public
lineTokens
:
ArrayBuffer
|
null
;
public
valid
:
boolean
;
constructor
(
state
:
IState
|
null
)
{
this
.
state
=
state
;
constructor
()
{
this
.
beginState
=
null
;
this
.
lineTokens
=
null
;
this
.
valid
=
false
;
}
...
...
@@ -167,18 +167,30 @@ class ModelLineTokens {
tokens
[
tokenIndex
<<
1
]
+=
textLength
;
}
}
public
setTokens
(
lineTokens
:
ArrayBuffer
|
null
,
valid
:
boolean
):
void
{
this
.
lineTokens
=
lineTokens
;
this
.
valid
=
valid
;
}
public
setBeginState
(
beginState
:
IState
|
null
):
void
{
this
.
beginState
=
beginState
;
}
public
invalidate
():
void
{
this
.
valid
=
false
;
}
}
export
interface
ITokensStore
{
_invalidLineStartIndex
:
number
;
readonly
invalidLineStartIndex
:
number
;
setGoodTokens
(
topLevelLanguageId
:
LanguageId
,
linesLength
:
number
,
lineIndex
:
number
,
lineTextLength
:
number
,
r
:
TokenizationResult2
):
void
;
setFakeTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
r
:
TokenizationResult2
):
void
;
getTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineText
:
string
):
LineTokens
;
invalidateLine
(
lineIndex
:
number
):
void
;
getState
(
lineIndex
:
number
):
IState
|
null
;
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
;
setGoodTokens
(
topLevelLanguageId
:
LanguageId
,
linesLength
:
number
,
lineIndex
:
number
,
text
:
string
,
r
:
TokenizationResult2
):
void
;
setState
(
lineIndex
:
number
,
state
:
IState
):
void
;
getBeginState
(
lineIndex
:
number
):
IState
|
null
;
applyEdits
(
range
:
Range
,
eolCount
:
number
,
firstLineLength
:
number
):
void
;
_getAllStates
(
linesLength
:
number
):
(
IState
|
null
)[];
...
...
@@ -187,7 +199,7 @@ export interface ITokensStore {
export
class
TokensStore
implements
ITokensStore
{
private
_tokens
:
ModelLineTokens
[];
_invalidLineStartIndex
:
number
;
private
_invalidLineStartIndex
:
number
;
private
_lastState
:
IState
|
null
;
constructor
(
initialState
:
IState
|
null
)
{
...
...
@@ -200,7 +212,7 @@ export class TokensStore implements ITokensStore {
this
.
_lastState
=
null
;
if
(
initialState
)
{
this
.
_
tokens
[
0
]
=
new
ModelLineTokens
(
initialState
);
this
.
_
setBeginState
(
0
,
initialState
);
}
}
...
...
@@ -210,7 +222,7 @@ export class TokensStore implements ITokensStore {
public
getTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineText
:
string
):
LineTokens
{
let
rawLineTokens
:
ArrayBuffer
|
null
=
null
;
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
]
)
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
rawLineTokens
=
this
.
_tokens
[
lineIndex
].
lineTokens
;
}
...
...
@@ -224,43 +236,31 @@ export class TokensStore implements ITokensStore {
return
new
LineTokens
(
lineTokens
,
lineText
);
}
public
invalidateLine
(
lineIndex
:
number
):
void
{
this
.
_setIsValid
(
lineIndex
,
false
);
if
(
lineIndex
<
this
.
_invalidLineStartIndex
)
{
this
.
_setIsValid
(
this
.
_invalidLineStartIndex
,
false
);
this
.
_invalidLineStartIndex
=
lineIndex
;
private
_invalidateLine
(
lineIndex
:
number
):
void
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
this
.
_tokens
[
lineIndex
].
invalidate
();
}
}
private
_setIsValid
(
lineIndex
:
number
,
valid
:
boolean
):
void
{
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
])
{
this
.
_tokens
[
lineIndex
].
valid
=
valid
;
if
(
lineIndex
<
this
.
_invalidLineStartIndex
)
{
this
.
_invalidLineStartIndex
=
lineIndex
;
}
}
private
_isValid
(
lineIndex
:
number
):
boolean
{
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
]
)
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
return
this
.
_tokens
[
lineIndex
].
valid
;
}
return
false
;
}
public
getState
(
lineIndex
:
number
):
IState
|
null
{
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
]
)
{
return
this
.
_tokens
[
lineIndex
].
s
tate
;
public
get
Begin
State
(
lineIndex
:
number
):
IState
|
null
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
return
this
.
_tokens
[
lineIndex
].
beginS
tate
;
}
return
null
;
}
public
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
let
target
:
ModelLineTokens
;
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
])
{
target
=
this
.
_tokens
[
lineIndex
];
}
else
{
target
=
new
ModelLineTokens
(
null
);
this
.
_tokens
[
lineIndex
]
=
target
;
}
private
static
_massageTokens
(
topLevelLanguageId
:
LanguageId
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
ArrayBuffer
{
if
(
lineTextLength
===
0
)
{
let
hasDifferentLanguageId
=
false
;
if
(
tokens
&&
tokens
.
length
>
1
)
{
...
...
@@ -268,8 +268,7 @@ export class TokensStore implements ITokensStore {
}
if
(
!
hasDifferentLanguageId
)
{
target
.
lineTokens
=
EMPTY_LINE_TOKENS
;
return
;
return
EMPTY_LINE_TOKENS
;
}
}
...
...
@@ -281,52 +280,62 @@ export class TokensStore implements ITokensStore {
LineTokens
.
convertToEndOffset
(
tokens
,
lineTextLength
);
target
.
lineTokens
=
tokens
.
buffer
;
}
public
setGoodTokens
(
topLevelLanguageId
:
LanguageId
,
linesLength
:
number
,
lineIndex
:
number
,
text
:
string
,
r
:
TokenizationResult2
):
void
{
const
endStateIndex
=
lineIndex
+
1
;
this
.
setTokens
(
topLevelLanguageId
,
lineIndex
,
text
.
length
,
r
.
tokens
);
this
.
_setIsValid
(
lineIndex
,
true
);
if
(
endStateIndex
<
linesLength
)
{
const
previousEndState
=
this
.
getState
(
endStateIndex
);
if
(
previousEndState
!==
null
&&
r
.
endState
.
equals
(
previousEndState
))
{
// The end state of this line remains the same
let
nextInvalidLineIndex
=
lineIndex
+
1
;
while
(
nextInvalidLineIndex
<
linesLength
)
{
if
(
!
this
.
_isValid
(
nextInvalidLineIndex
))
{
break
;
}
if
(
nextInvalidLineIndex
+
1
<
linesLength
)
{
if
(
this
.
getState
(
nextInvalidLineIndex
+
1
)
===
null
)
{
break
;
}
}
else
{
if
(
this
.
_lastState
===
null
)
{
break
;
}
}
nextInvalidLineIndex
++
;
}
this
.
_invalidLineStartIndex
=
nextInvalidLineIndex
;
}
else
{
this
.
_invalidLineStartIndex
=
lineIndex
+
1
;
this
.
setState
(
endStateIndex
,
r
.
endState
);
}
}
else
{
this
.
_lastState
=
r
.
endState
;
this
.
_invalidLineStartIndex
=
linesLength
;
return
tokens
.
buffer
;
}
private
_getOrCreate
(
lineIndex
:
number
):
ModelLineTokens
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
return
this
.
_tokens
[
lineIndex
];
}
while
(
lineIndex
>
this
.
_tokens
.
length
)
{
this
.
_tokens
[
this
.
_tokens
.
length
]
=
new
ModelLineTokens
();
}
const
result
=
new
ModelLineTokens
();
this
.
_tokens
[
lineIndex
]
=
result
;
return
result
;
}
public
setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
if
(
lineIndex
<
this
.
_tokens
.
length
&&
this
.
_tokens
[
lineIndex
])
{
this
.
_tokens
[
lineIndex
].
state
=
state
;
}
else
{
const
tmp
=
new
ModelLineTokens
(
state
);
this
.
_tokens
[
lineIndex
]
=
tmp
;
private
_setTokens
(
lineIndex
:
number
,
tokens
:
ArrayBuffer
|
null
,
valid
:
boolean
):
void
{
this
.
_getOrCreate
(
lineIndex
).
setTokens
(
tokens
,
valid
);
}
private
_setBeginState
(
lineIndex
:
number
,
beginState
:
IState
|
null
):
void
{
this
.
_getOrCreate
(
lineIndex
).
setBeginState
(
beginState
);
}
public
setGoodTokens
(
topLevelLanguageId
:
LanguageId
,
linesLength
:
number
,
lineIndex
:
number
,
lineTextLength
:
number
,
r
:
TokenizationResult2
):
void
{
const
tokens
=
TokensStore
.
_massageTokens
(
topLevelLanguageId
,
lineTextLength
,
r
.
tokens
);
this
.
_setTokens
(
lineIndex
,
tokens
,
true
);
this
.
_invalidLineStartIndex
=
lineIndex
+
1
;
// Check if this was the last line
if
(
lineIndex
===
linesLength
-
1
)
{
this
.
_lastState
=
r
.
endState
;
return
;
}
// Check if the end state has changed
const
previousEndState
=
this
.
getBeginState
(
lineIndex
+
1
);
if
(
previousEndState
===
null
||
!
r
.
endState
.
equals
(
previousEndState
))
{
this
.
_setBeginState
(
lineIndex
+
1
,
r
.
endState
);
this
.
_invalidateLine
(
lineIndex
+
1
);
return
;
}
// Perhaps we can skip tokenizing some lines...
let
i
=
lineIndex
+
1
;
while
(
i
<
linesLength
)
{
if
(
!
this
.
_isValid
(
i
))
{
break
;
}
i
++
;
}
this
.
_invalidLineStartIndex
=
i
;
}
setFakeTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
r
:
TokenizationResult2
):
void
{
const
tokens
=
TokensStore
.
_massageTokens
(
topLevelLanguageId
,
lineTextLength
,
r
.
tokens
);
this
.
_setTokens
(
lineIndex
,
tokens
,
false
);
}
//#region Editing
...
...
@@ -338,14 +347,14 @@ export class TokensStore implements ITokensStore {
const
editingLinesCnt
=
Math
.
min
(
deletingLinesCnt
,
insertingLinesCnt
);
for
(
let
j
=
editingLinesCnt
;
j
>=
0
;
j
--
)
{
this
.
invalidateLine
(
range
.
startLineNumber
+
j
-
1
);
this
.
_
invalidateLine
(
range
.
startLineNumber
+
j
-
1
);
}
this
.
_acceptDeleteRange
(
range
);
this
.
_acceptInsertText
(
new
Position
(
range
.
startLineNumber
,
range
.
startColumn
),
eolCount
,
firstLineLength
);
}
catch
(
err
)
{
// emergency recovery => reset tokens
this
.
_reset
(
this
.
getState
(
0
));
this
.
_reset
(
this
.
get
Begin
State
(
0
));
}
}
...
...
@@ -407,8 +416,8 @@ export class TokensStore implements ITokensStore {
line
.
insert
(
position
.
column
-
1
,
firstLineLength
);
let
insert
:
ModelLineTokens
[]
=
new
Array
<
ModelLineTokens
>
(
eolCount
);
for
(
let
i
=
eolCount
-
1
;
i
>=
0
;
i
--
)
{
insert
[
i
]
=
new
ModelLineTokens
(
null
);
for
(
let
i
=
0
;
i
<
eolCount
;
i
++
)
{
insert
[
i
]
=
new
ModelLineTokens
();
}
this
.
_tokens
=
arrays
.
arrayInsert
(
this
.
_tokens
,
position
.
lineNumber
,
insert
);
}
...
...
@@ -418,7 +427,7 @@ export class TokensStore implements ITokensStore {
_getAllStates
(
linesLength
:
number
):
(
IState
|
null
)[]
{
const
r
:
(
IState
|
null
)[]
=
[];
for
(
let
i
=
0
;
i
<
linesLength
;
i
++
)
{
r
[
i
]
=
this
.
getState
(
i
);
r
[
i
]
=
this
.
get
Begin
State
(
i
);
}
r
[
linesLength
]
=
this
.
_lastState
;
return
r
;
...
...
@@ -457,6 +466,10 @@ export class ModelLinesTokens implements IModelLinesTokens {
}
public
isCheapToTokenize
(
store
:
ITokensStore
,
buffer
:
ITextBuffer
,
lineNumber
:
number
):
boolean
{
if
(
!
this
.
tokenizationSupport
)
{
return
true
;
}
const
firstInvalidLineNumber
=
store
.
invalidLineStartIndex
+
1
;
if
(
lineNumber
>
firstInvalidLineNumber
)
{
return
false
;
...
...
@@ -474,6 +487,10 @@ export class ModelLinesTokens implements IModelLinesTokens {
}
public
hasLinesToTokenize
(
store
:
ITokensStore
,
buffer
:
ITextBuffer
):
boolean
{
if
(
!
this
.
tokenizationSupport
)
{
return
false
;
}
return
(
store
.
invalidLineStartIndex
<
buffer
.
getLineCount
());
}
...
...
@@ -490,7 +507,6 @@ export class ModelLinesTokens implements IModelLinesTokens {
public
updateTokensUntilLine
(
store
:
ITokensStore
,
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
lineNumber
:
number
):
void
{
if
(
!
this
.
tokenizationSupport
)
{
store
.
_invalidLineStartIndex
=
buffer
.
getLineCount
();
return
;
}
...
...
@@ -500,10 +516,10 @@ export class ModelLinesTokens implements IModelLinesTokens {
// Validate all states up to and including endLineIndex
for
(
let
lineIndex
=
store
.
invalidLineStartIndex
;
lineIndex
<=
endLineIndex
;
lineIndex
++
)
{
const
text
=
buffer
.
getLineContent
(
lineIndex
+
1
);
const
lineStartState
=
store
.
getState
(
lineIndex
);
const
lineStartState
=
store
.
get
Begin
State
(
lineIndex
);
const
r
=
safeTokenize
(
this
.
_languageIdentifier
,
this
.
tokenizationSupport
,
text
,
lineStartState
!
);
store
.
setGoodTokens
(
this
.
_languageIdentifier
.
id
,
linesLength
,
lineIndex
,
text
,
r
);
store
.
setGoodTokens
(
this
.
_languageIdentifier
.
id
,
linesLength
,
lineIndex
,
text
.
length
,
r
);
eventBuilder
.
registerChangedTokens
(
lineIndex
+
1
);
lineIndex
=
store
.
invalidLineStartIndex
-
1
;
// -1 because the outer loop increments it
}
...
...
@@ -537,7 +553,7 @@ export class ModelLinesTokens implements IModelLinesTokens {
}
if
(
newNonWhitespaceIndex
<
nonWhitespaceColumn
)
{
initialState
=
store
.
getState
(
i
-
1
);
initialState
=
store
.
get
Begin
State
(
i
-
1
);
if
(
initialState
)
{
break
;
}
...
...
@@ -553,37 +569,15 @@ export class ModelLinesTokens implements IModelLinesTokens {
let
state
=
initialState
;
for
(
let
i
=
fakeLines
.
length
-
1
;
i
>=
0
;
i
--
)
{
let
r
=
safeTokenize
(
this
.
_languageIdentifier
,
this
.
tokenizationSupport
,
fakeLines
[
i
],
state
);
if
(
r
)
{
state
=
r
.
endState
;
}
else
{
state
=
initialState
;
}
state
=
r
.
endState
;
}
this
.
_fakeTokenizeLines
(
store
,
buffer
,
eventBuilder
,
state
,
startLineNumber
,
endLineNumber
);
}
private
_fakeTokenizeLines
(
store
:
ITokensStore
,
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
initialState
:
IState
,
startLineNumber
:
number
,
endLineNumber
:
number
):
void
{
if
(
!
this
.
tokenizationSupport
)
{
return
;
}
let
state
=
initialState
;
for
(
let
i
=
startLineNumber
;
i
<=
endLineNumber
;
i
++
)
{
let
text
=
buffer
.
getLineContent
(
i
);
for
(
let
lineNumber
=
startLineNumber
;
lineNumber
<=
endLineNumber
;
lineNumber
++
)
{
let
text
=
buffer
.
getLineContent
(
lineNumber
);
let
r
=
safeTokenize
(
this
.
_languageIdentifier
,
this
.
tokenizationSupport
,
text
,
state
);
if
(
r
)
{
store
.
setTokens
(
this
.
_languageIdentifier
.
id
,
i
-
1
,
text
.
length
,
r
.
tokens
);
// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
store
.
invalidateLine
(
i
-
1
);
store
.
setState
(
i
-
1
,
state
);
state
=
r
.
endState
;
eventBuilder
.
registerChangedTokens
(
i
);
}
else
{
state
=
initialState
;
}
store
.
setFakeTokens
(
this
.
_languageIdentifier
.
id
,
lineNumber
-
1
,
text
.
length
,
r
);
state
=
r
.
endState
;
eventBuilder
.
registerChangedTokens
(
lineNumber
);
}
}
...
...
src/vs/editor/test/common/model/model.line.test.ts
浏览文件 @
7f833a91
...
...
@@ -9,6 +9,8 @@ import { Range } from 'vs/editor/common/core/range';
import
{
TextModel
}
from
'
vs/editor/common/model/textModel
'
;
import
{
LanguageIdentifier
,
MetadataConsts
}
from
'
vs/editor/common/modes
'
;
import
{
ViewLineToken
,
ViewLineTokenFactory
}
from
'
vs/editor/test/common/core/viewLineToken
'
;
import
{
TokenizationResult2
}
from
'
vs/editor/common/core/token
'
;
import
{
NULL_STATE
}
from
'
vs/editor/common/modes/nullMode
'
;
interface
ILineEdit
{
startColumn
:
number
;
...
...
@@ -92,6 +94,10 @@ class TestToken {
}
}
function
toTokenizationResult2
(
tokens
:
Uint32Array
):
TokenizationResult2
{
return
new
TokenizationResult2
(
tokens
,
NULL_STATE
);
}
suite
(
'
ModelLinesTokens
'
,
()
=>
{
interface
IBufferLineState
{
...
...
@@ -110,7 +116,7 @@ suite('ModelLinesTokens', () => {
for
(
let
lineIndex
=
0
;
lineIndex
<
initial
.
length
;
lineIndex
++
)
{
const
lineTokens
=
initial
[
lineIndex
].
tokens
;
const
lineTextLength
=
model
.
getLineMaxColumn
(
lineIndex
+
1
)
-
1
;
model
.
_tokens
.
set
Tokens
(
0
,
lineIndex
,
lineTextLength
,
TestToken
.
toTokens
(
lineTokens
));
model
.
_tokens
.
set
FakeTokens
(
0
,
lineIndex
,
lineTextLength
,
toTokenizationResult2
(
TestToken
.
toTokens
(
lineTokens
)
));
}
model
.
applyEdits
(
edits
.
map
((
ed
)
=>
({
...
...
@@ -441,14 +447,14 @@ suite('ModelLinesTokens', () => {
test
(
'
insertion on empty line
'
,
()
=>
{
const
model
=
new
TextModel
(
'
some text
'
,
TextModel
.
DEFAULT_CREATION_OPTIONS
,
new
LanguageIdentifier
(
'
test
'
,
0
));
model
.
_tokens
.
set
Tokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
TestToken
.
toTokens
([
new
TestToken
(
0
,
1
)]
));
model
.
_tokens
.
set
FakeTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
toTokenizationResult2
(
TestToken
.
toTokens
([
new
TestToken
(
0
,
1
)])
));
model
.
applyEdits
([{
range
:
new
Range
(
1
,
1
,
1
,
10
),
text
:
''
}]);
model
.
_tokens
.
set
Tokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
new
Uint32Array
(
0
));
model
.
_tokens
.
set
FakeTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
toTokenizationResult2
(
new
Uint32Array
(
0
)
));
model
.
applyEdits
([{
range
:
new
Range
(
1
,
1
,
1
,
1
),
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录