Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
掘金者说
vscode
提交
d1c16cff
V
vscode
项目概览
掘金者说
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d1c16cff
编写于
7月 03, 2019
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Reduce IModelLinesTokens shape
上级
8a5f705e
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
80 addition
and
108 deletion
+80
-108
src/vs/editor/common/model/textModel.ts
src/vs/editor/common/model/textModel.ts
+5
-55
src/vs/editor/common/model/textModelTokens.ts
src/vs/editor/common/model/textModelTokens.ts
+72
-50
src/vs/editor/test/common/model/model.line.test.ts
src/vs/editor/test/common/model/model.line.test.ts
+3
-3
未找到文件。
src/vs/editor/common/model/textModel.ts
浏览文件 @
d1c16cff
...
...
@@ -23,9 +23,9 @@ import { IntervalNode, IntervalTree, getNodeIsInOverviewRuler, recomputeMaxEnd }
import
{
PieceTreeTextBufferBuilder
}
from
'
vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder
'
;
import
{
IModelContentChangedEvent
,
IModelDecorationsChangedEvent
,
IModelLanguageChangedEvent
,
IModelLanguageConfigurationChangedEvent
,
IModelOptionsChangedEvent
,
IModelTokensChangedEvent
,
InternalModelContentChangeEvent
,
ModelRawChange
,
ModelRawContentChangedEvent
,
ModelRawEOLChanged
,
ModelRawFlush
,
ModelRawLineChanged
,
ModelRawLinesDeleted
,
ModelRawLinesInserted
}
from
'
vs/editor/common/model/textModelEvents
'
;
import
{
SearchData
,
SearchParams
,
TextModelSearch
}
from
'
vs/editor/common/model/textModelSearch
'
;
import
{
ModelLinesTokens
,
ModelTokensChangedEventBuilder
,
IModelLinesTokens
,
safeTokenize
}
from
'
vs/editor/common/model/textModelTokens
'
;
import
{
ModelLinesTokens
,
ModelTokensChangedEventBuilder
,
IModelLinesTokens
}
from
'
vs/editor/common/model/textModelTokens
'
;
import
{
getWordAtText
}
from
'
vs/editor/common/model/wordHelper
'
;
import
{
IState
,
LanguageId
,
LanguageIdentifier
,
TokenizationRegistry
,
FormattingOptions
}
from
'
vs/editor/common/modes
'
;
import
{
LanguageId
,
LanguageIdentifier
,
TokenizationRegistry
,
FormattingOptions
}
from
'
vs/editor/common/modes
'
;
import
{
LanguageConfigurationRegistry
}
from
'
vs/editor/common/modes/languageConfigurationRegistry
'
;
import
{
NULL_LANGUAGE_IDENTIFIER
}
from
'
vs/editor/common/modes/nullMode
'
;
import
{
ignoreBracketsInToken
}
from
'
vs/editor/common/modes/supports
'
;
...
...
@@ -1342,7 +1342,7 @@ export class TextModel extends Disposable implements model.ITextModel {
const
change
=
contentChanges
[
i
];
const
[
eolCount
,
firstLineLength
]
=
TextModel
.
_eolCount
(
change
.
text
);
try
{
this
.
_tokens
.
applyEdits
(
change
.
range
,
eolCount
,
firstLineLength
);
this
.
_tokens
.
store
.
applyEdits
(
change
.
range
,
eolCount
,
firstLineLength
);
}
catch
(
err
)
{
// emergency recovery => reset tokens
this
.
_tokens
=
new
ModelLinesTokens
(
this
.
_languageIdentifier
,
this
.
_tokens
.
tokenizationSupport
);
...
...
@@ -1776,61 +1776,11 @@ export class TextModel extends Disposable implements model.ITextModel {
//#region Tokenization
public
tokenizeViewport
(
startLineNumber
:
number
,
endLineNumber
:
number
):
void
{
if
(
!
this
.
_tokens
.
tokenizationSupport
)
{
// nothing to do
return
;
}
startLineNumber
=
Math
.
max
(
1
,
startLineNumber
);
endLineNumber
=
Math
.
min
(
this
.
getLineCount
(),
endLineNumber
);
if
(
endLineNumber
<=
this
.
_tokens
.
invalidLineStartIndex
)
{
// nothing to do
return
;
}
if
(
startLineNumber
<=
this
.
_tokens
.
invalidLineStartIndex
)
{
// tokenization has reached the viewport start...
this
.
forceTokenization
(
endLineNumber
);
return
;
}
let
nonWhitespaceColumn
=
this
.
getLineFirstNonWhitespaceColumn
(
startLineNumber
);
let
fakeLines
:
string
[]
=
[];
let
initialState
:
IState
|
null
=
null
;
for
(
let
i
=
startLineNumber
-
1
;
nonWhitespaceColumn
>
0
&&
i
>=
1
;
i
--
)
{
let
newNonWhitespaceIndex
=
this
.
getLineFirstNonWhitespaceColumn
(
i
);
if
(
newNonWhitespaceIndex
===
0
)
{
continue
;
}
if
(
newNonWhitespaceIndex
<
nonWhitespaceColumn
)
{
initialState
=
this
.
_tokens
.
getState
(
i
-
1
);
if
(
initialState
)
{
break
;
}
fakeLines
.
push
(
this
.
getLineContent
(
i
));
nonWhitespaceColumn
=
newNonWhitespaceIndex
;
}
}
if
(
!
initialState
)
{
initialState
=
this
.
_tokens
.
tokenizationSupport
.
getInitialState
();
}
let
state
=
initialState
;
for
(
let
i
=
fakeLines
.
length
-
1
;
i
>=
0
;
i
--
)
{
let
r
=
safeTokenize
(
this
.
_languageIdentifier
,
this
.
_tokens
.
tokenizationSupport
,
fakeLines
[
i
],
state
);
if
(
r
)
{
state
=
r
.
endState
;
}
else
{
state
=
initialState
;
}
}
const
eventBuilder
=
new
ModelTokensChangedEventBuilder
();
this
.
_tokens
.
fakeTokenizeLines
(
this
.
_buffer
,
eventBuilder
,
state
,
startLineNumber
,
endLineNumber
);
this
.
_tokens
.
tokenizeViewport
(
this
.
_buffer
,
eventBuilder
,
startLineNumber
,
endLineNumber
);
const
e
=
eventBuilder
.
build
();
if
(
e
)
{
...
...
@@ -1896,7 +1846,7 @@ export class TextModel extends Disposable implements model.ITextModel {
private
_getLineTokens
(
lineNumber
:
number
):
LineTokens
{
const
lineText
=
this
.
_buffer
.
getLineContent
(
lineNumber
);
return
this
.
_tokens
.
getTokens
(
this
.
_languageIdentifier
.
id
,
lineNumber
-
1
,
lineText
);
return
this
.
_tokens
.
store
.
getTokens
(
this
.
_languageIdentifier
.
id
,
lineNumber
-
1
,
lineText
);
}
public
getLanguageIdentifier
():
LanguageIdentifier
{
...
...
src/vs/editor/common/model/textModelTokens.ts
浏览文件 @
d1c16cff
...
...
@@ -390,19 +390,15 @@ class TokensStore {
export
interface
IModelLinesTokens
{
readonly
tokenizationSupport
:
ITokenizationSupport
|
null
;
readonly
store
:
TokensStore
;
readonly
invalidLineStartIndex
:
number
;
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
;
getTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineText
:
string
):
LineTokens
;
isCheapToTokenize
(
lineNumber
:
number
):
boolean
;
hasLinesToTokenize
(
buffer
:
ITextBuffer
):
boolean
;
getState
(
lineIndex
:
number
):
IState
|
null
;
applyEdits
(
range
:
Range
,
eolCount
:
number
,
firstLineLength
:
number
):
void
;
tokenizeOneInvalidLine
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
):
number
;
updateTokensUntilLine
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
lineNumber
:
number
):
void
;
fakeTokenizeLines
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
initialState
:
IState
,
startLineNumber
:
number
,
endLineNumber
:
number
):
void
;
tokenizeViewport
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
startLineNumber
:
number
,
endLineNumber
:
number
):
void
;
_getAllStates
(
linesLength
:
number
):
(
IState
|
null
)[];
_getAllInvalid
(
linesLength
:
number
):
number
[];
...
...
@@ -410,12 +406,16 @@ export interface IModelLinesTokens {
export
class
ModelLinesTokens
implements
IModelLinesTokens
{
p
ublic
readonly
languageIdentifier
:
LanguageIdentifier
;
p
rivate
readonly
_
languageIdentifier
:
LanguageIdentifier
;
public
readonly
tokenizationSupport
:
ITokenizationSupport
|
null
;
public
readonly
store
:
TokensStore
;
public
get
invalidLineStartIndex
()
{
return
this
.
store
.
invalidLineStartIndex
;
}
constructor
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
|
null
)
{
this
.
languageIdentifier
=
languageIdentifier
;
this
.
_
languageIdentifier
=
languageIdentifier
;
this
.
tokenizationSupport
=
tokenizationSupport
;
let
initialState
:
IState
|
null
=
null
;
...
...
@@ -430,14 +430,6 @@ export class ModelLinesTokens implements IModelLinesTokens {
this
.
store
=
new
TokensStore
(
initialState
);
}
public
get
invalidLineStartIndex
()
{
return
this
.
store
.
invalidLineStartIndex
;
}
public
getTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineText
:
string
):
LineTokens
{
return
this
.
store
.
getTokens
(
topLevelLanguageId
,
lineIndex
,
lineText
);
}
public
isCheapToTokenize
(
lineNumber
:
number
):
boolean
{
const
firstInvalidLineNumber
=
this
.
store
.
invalidLineStartIndex
+
1
;
return
(
firstInvalidLineNumber
>=
lineNumber
);
...
...
@@ -447,30 +439,6 @@ export class ModelLinesTokens implements IModelLinesTokens {
return
(
this
.
store
.
invalidLineStartIndex
<
buffer
.
getLineCount
());
}
public
getState
(
lineIndex
:
number
):
IState
|
null
{
return
this
.
store
.
getState
(
lineIndex
);
}
public
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
this
.
store
.
setTokens
(
topLevelLanguageId
,
lineIndex
,
lineTextLength
,
tokens
);
}
private
_setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
this
.
store
.
setState
(
lineIndex
,
state
);
}
//#region Editing
public
applyEdits
(
range
:
Range
,
eolCount
:
number
,
firstLineLength
:
number
):
void
{
this
.
store
.
applyEdits
(
range
,
eolCount
,
firstLineLength
);
}
private
_invalidateLine
(
lineIndex
:
number
):
void
{
this
.
store
.
invalidateLine
(
lineIndex
);
}
//#endregion
//#region Tokenization
public
tokenizeOneInvalidLine
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
):
number
{
...
...
@@ -494,16 +462,70 @@ export class ModelLinesTokens implements IModelLinesTokens {
// Validate all states up to and including endLineIndex
for
(
let
lineIndex
=
this
.
store
.
invalidLineStartIndex
;
lineIndex
<=
endLineIndex
;
lineIndex
++
)
{
const
text
=
buffer
.
getLineContent
(
lineIndex
+
1
);
const
lineStartState
=
this
.
getState
(
lineIndex
);
const
lineStartState
=
this
.
store
.
getState
(
lineIndex
);
const
r
=
safeTokenize
(
this
.
languageIdentifier
,
this
.
tokenizationSupport
,
text
,
lineStartState
!
);
this
.
store
.
setGoodTokens
(
this
.
languageIdentifier
.
id
,
linesLength
,
lineIndex
,
text
,
r
);
const
r
=
safeTokenize
(
this
.
_
languageIdentifier
,
this
.
tokenizationSupport
,
text
,
lineStartState
!
);
this
.
store
.
setGoodTokens
(
this
.
_
languageIdentifier
.
id
,
linesLength
,
lineIndex
,
text
,
r
);
eventBuilder
.
registerChangedTokens
(
lineIndex
+
1
);
lineIndex
=
this
.
store
.
invalidLineStartIndex
-
1
;
// -1 because the outer loop increments it
}
}
public
fakeTokenizeLines
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
initialState
:
IState
,
startLineNumber
:
number
,
endLineNumber
:
number
):
void
{
public
tokenizeViewport
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
startLineNumber
:
number
,
endLineNumber
:
number
):
void
{
if
(
!
this
.
tokenizationSupport
)
{
// nothing to do
return
;
}
if
(
endLineNumber
<=
this
.
invalidLineStartIndex
)
{
// nothing to do
return
;
}
if
(
startLineNumber
<=
this
.
invalidLineStartIndex
)
{
// tokenization has reached the viewport start...
this
.
updateTokensUntilLine
(
buffer
,
eventBuilder
,
endLineNumber
);
return
;
}
let
nonWhitespaceColumn
=
buffer
.
getLineFirstNonWhitespaceColumn
(
startLineNumber
);
let
fakeLines
:
string
[]
=
[];
let
initialState
:
IState
|
null
=
null
;
for
(
let
i
=
startLineNumber
-
1
;
nonWhitespaceColumn
>
0
&&
i
>=
1
;
i
--
)
{
let
newNonWhitespaceIndex
=
buffer
.
getLineFirstNonWhitespaceColumn
(
i
);
if
(
newNonWhitespaceIndex
===
0
)
{
continue
;
}
if
(
newNonWhitespaceIndex
<
nonWhitespaceColumn
)
{
initialState
=
this
.
store
.
getState
(
i
-
1
);
if
(
initialState
)
{
break
;
}
fakeLines
.
push
(
buffer
.
getLineContent
(
i
));
nonWhitespaceColumn
=
newNonWhitespaceIndex
;
}
}
if
(
!
initialState
)
{
initialState
=
this
.
tokenizationSupport
.
getInitialState
();
}
let
state
=
initialState
;
for
(
let
i
=
fakeLines
.
length
-
1
;
i
>=
0
;
i
--
)
{
let
r
=
safeTokenize
(
this
.
_languageIdentifier
,
this
.
tokenizationSupport
,
fakeLines
[
i
],
state
);
if
(
r
)
{
state
=
r
.
endState
;
}
else
{
state
=
initialState
;
}
}
this
.
_fakeTokenizeLines
(
buffer
,
eventBuilder
,
state
,
startLineNumber
,
endLineNumber
);
}
private
_fakeTokenizeLines
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
initialState
:
IState
,
startLineNumber
:
number
,
endLineNumber
:
number
):
void
{
if
(
!
this
.
tokenizationSupport
)
{
return
;
}
...
...
@@ -511,14 +533,14 @@ export class ModelLinesTokens implements IModelLinesTokens {
let
state
=
initialState
;
for
(
let
i
=
startLineNumber
;
i
<=
endLineNumber
;
i
++
)
{
let
text
=
buffer
.
getLineContent
(
i
);
let
r
=
safeTokenize
(
this
.
languageIdentifier
,
this
.
tokenizationSupport
,
text
,
state
);
let
r
=
safeTokenize
(
this
.
_
languageIdentifier
,
this
.
tokenizationSupport
,
text
,
state
);
if
(
r
)
{
this
.
s
etTokens
(
this
.
languageIdentifier
.
id
,
i
-
1
,
text
.
length
,
r
.
tokens
);
this
.
s
tore
.
setTokens
(
this
.
_
languageIdentifier
.
id
,
i
-
1
,
text
.
length
,
r
.
tokens
);
// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
this
.
_
invalidateLine
(
i
-
1
);
this
.
_
setState
(
i
-
1
,
state
);
this
.
store
.
invalidateLine
(
i
-
1
);
this
.
store
.
setState
(
i
-
1
,
state
);
state
=
r
.
endState
;
eventBuilder
.
registerChangedTokens
(
i
);
}
else
{
...
...
@@ -532,7 +554,7 @@ export class ModelLinesTokens implements IModelLinesTokens {
_getAllStates
(
linesLength
:
number
):
(
IState
|
null
)[]
{
const
r
:
(
IState
|
null
)[]
=
[];
for
(
let
i
=
0
;
i
<
linesLength
;
i
++
)
{
r
[
i
]
=
this
.
getState
(
i
);
r
[
i
]
=
this
.
store
.
getState
(
i
);
}
r
[
linesLength
]
=
this
.
store
.
_lastState
;
return
r
;
...
...
@@ -549,7 +571,7 @@ export class ModelLinesTokens implements IModelLinesTokens {
}
}
export
function
safeTokenize
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
|
null
,
text
:
string
,
state
:
IState
):
TokenizationResult2
{
function
safeTokenize
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
|
null
,
text
:
string
,
state
:
IState
):
TokenizationResult2
{
let
r
:
TokenizationResult2
|
null
=
null
;
if
(
tokenizationSupport
)
{
...
...
src/vs/editor/test/common/model/model.line.test.ts
浏览文件 @
d1c16cff
...
...
@@ -110,7 +110,7 @@ suite('ModelLinesTokens', () => {
for
(
let
lineIndex
=
0
;
lineIndex
<
initial
.
length
;
lineIndex
++
)
{
const
lineTokens
=
initial
[
lineIndex
].
tokens
;
const
lineTextLength
=
model
.
getLineMaxColumn
(
lineIndex
+
1
)
-
1
;
model
.
_tokens
.
setTokens
(
0
,
lineIndex
,
lineTextLength
,
TestToken
.
toTokens
(
lineTokens
));
model
.
_tokens
.
s
tore
.
s
etTokens
(
0
,
lineIndex
,
lineTextLength
,
TestToken
.
toTokens
(
lineTokens
));
}
model
.
applyEdits
(
edits
.
map
((
ed
)
=>
({
...
...
@@ -441,14 +441,14 @@ suite('ModelLinesTokens', () => {
test
(
'
insertion on empty line
'
,
()
=>
{
const
model
=
new
TextModel
(
'
some text
'
,
TextModel
.
DEFAULT_CREATION_OPTIONS
,
new
LanguageIdentifier
(
'
test
'
,
0
));
model
.
_tokens
.
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
TestToken
.
toTokens
([
new
TestToken
(
0
,
1
)]));
model
.
_tokens
.
s
tore
.
s
etTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
TestToken
.
toTokens
([
new
TestToken
(
0
,
1
)]));
model
.
applyEdits
([{
range
:
new
Range
(
1
,
1
,
1
,
10
),
text
:
''
}]);
model
.
_tokens
.
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
new
Uint32Array
(
0
));
model
.
_tokens
.
s
tore
.
s
etTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
new
Uint32Array
(
0
));
model
.
applyEdits
([{
range
:
new
Range
(
1
,
1
,
1
,
1
),
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录