Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
掘金者说
vscode
提交
8a5f705e
V
vscode
项目概览
掘金者说
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
8a5f705e
编写于
7月 03, 2019
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Extract IModelLinesTokens
上级
9f20ed4d
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
109 addition
and
84 deletion
+109
-84
src/vs/editor/common/model/textModel.ts
src/vs/editor/common/model/textModel.ts
+11
-26
src/vs/editor/common/model/textModelTokens.ts
src/vs/editor/common/model/textModelTokens.ts
+90
-43
src/vs/editor/test/common/model/model.line.test.ts
src/vs/editor/test/common/model/model.line.test.ts
+3
-3
src/vs/editor/test/common/model/model.modes.test.ts
src/vs/editor/test/common/model/model.modes.test.ts
+5
-12
未找到文件。
src/vs/editor/common/model/textModel.ts
浏览文件 @
8a5f705e
...
...
@@ -23,7 +23,7 @@ import { IntervalNode, IntervalTree, getNodeIsInOverviewRuler, recomputeMaxEnd }
import
{
PieceTreeTextBufferBuilder
}
from
'
vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder
'
;
import
{
IModelContentChangedEvent
,
IModelDecorationsChangedEvent
,
IModelLanguageChangedEvent
,
IModelLanguageConfigurationChangedEvent
,
IModelOptionsChangedEvent
,
IModelTokensChangedEvent
,
InternalModelContentChangeEvent
,
ModelRawChange
,
ModelRawContentChangedEvent
,
ModelRawEOLChanged
,
ModelRawFlush
,
ModelRawLineChanged
,
ModelRawLinesDeleted
,
ModelRawLinesInserted
}
from
'
vs/editor/common/model/textModelEvents
'
;
import
{
SearchData
,
SearchParams
,
TextModelSearch
}
from
'
vs/editor/common/model/textModelSearch
'
;
import
{
ModelLinesTokens
,
ModelTokensChangedEventBuilder
}
from
'
vs/editor/common/model/textModelTokens
'
;
import
{
ModelLinesTokens
,
ModelTokensChangedEventBuilder
,
IModelLinesTokens
,
safeTokenize
}
from
'
vs/editor/common/model/textModelTokens
'
;
import
{
getWordAtText
}
from
'
vs/editor/common/model/wordHelper
'
;
import
{
IState
,
LanguageId
,
LanguageIdentifier
,
TokenizationRegistry
,
FormattingOptions
}
from
'
vs/editor/common/modes
'
;
import
{
LanguageConfigurationRegistry
}
from
'
vs/editor/common/modes/languageConfigurationRegistry
'
;
...
...
@@ -287,7 +287,7 @@ export class TextModel extends Disposable implements model.ITextModel {
private
readonly
_tokenizationListener
:
IDisposable
;
private
readonly
_languageRegistryListener
:
IDisposable
;
private
_revalidateTokensTimeout
:
any
;
/*private*/
_tokens
:
ModelLinesTokens
;
/*private*/
_tokens
:
I
ModelLinesTokens
;
//#endregion
constructor
(
source
:
string
|
model
.
ITextBufferFactory
,
creationOptions
:
model
.
ITextModelCreationOptions
,
languageIdentifier
:
LanguageIdentifier
|
null
,
associatedResource
:
URI
|
null
=
null
)
{
...
...
@@ -1345,7 +1345,7 @@ export class TextModel extends Disposable implements model.ITextModel {
this
.
_tokens
.
applyEdits
(
change
.
range
,
eolCount
,
firstLineLength
);
}
catch
(
err
)
{
// emergency recovery => reset tokens
this
.
_tokens
=
new
ModelLinesTokens
(
this
.
_
tokens
.
languageIdentifier
,
this
.
_tokens
.
tokenizationSupport
);
this
.
_tokens
=
new
ModelLinesTokens
(
this
.
_languageIdentifier
,
this
.
_tokens
.
tokenizationSupport
);
}
this
.
_onDidChangeDecorations
.
fire
();
this
.
_decorationsTree
.
acceptReplace
(
change
.
rangeOffset
,
change
.
rangeLength
,
change
.
text
.
length
,
change
.
forceMoveMarkers
);
...
...
@@ -1806,7 +1806,7 @@ export class TextModel extends Disposable implements model.ITextModel {
}
if
(
newNonWhitespaceIndex
<
nonWhitespaceColumn
)
{
initialState
=
this
.
_tokens
.
_
getState
(
i
-
1
);
initialState
=
this
.
_tokens
.
getState
(
i
-
1
);
if
(
initialState
)
{
break
;
}
...
...
@@ -1819,33 +1819,18 @@ export class TextModel extends Disposable implements model.ITextModel {
initialState
=
this
.
_tokens
.
tokenizationSupport
.
getInitialState
();
}
let
state
=
initialState
.
clone
()
;
let
state
=
initialState
;
for
(
let
i
=
fakeLines
.
length
-
1
;
i
>=
0
;
i
--
)
{
let
r
=
this
.
_tokens
.
_tokenizeText
(
this
.
_buffer
,
fakeLines
[
i
],
state
);
let
r
=
safeTokenize
(
this
.
_languageIdentifier
,
this
.
_tokens
.
tokenizationSupport
,
fakeLines
[
i
],
state
);
if
(
r
)
{
state
=
r
.
endState
.
clone
()
;
state
=
r
.
endState
;
}
else
{
state
=
initialState
.
clone
()
;
state
=
initialState
;
}
}
const
eventBuilder
=
new
ModelTokensChangedEventBuilder
();
for
(
let
i
=
startLineNumber
;
i
<=
endLineNumber
;
i
++
)
{
let
text
=
this
.
getLineContent
(
i
);
let
r
=
this
.
_tokens
.
_tokenizeText
(
this
.
_buffer
,
text
,
state
);
if
(
r
)
{
this
.
_tokens
.
_setTokens
(
this
.
_tokens
.
languageIdentifier
.
id
,
i
-
1
,
text
.
length
,
r
.
tokens
);
// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
this
.
_tokens
.
_invalidateLine
(
i
-
1
);
this
.
_tokens
.
_setState
(
i
-
1
,
state
);
state
=
r
.
endState
.
clone
();
eventBuilder
.
registerChangedTokens
(
i
);
}
else
{
state
=
initialState
.
clone
();
}
}
this
.
_tokens
.
fakeTokenizeLines
(
this
.
_buffer
,
eventBuilder
,
state
,
startLineNumber
,
endLineNumber
);
const
e
=
eventBuilder
.
build
();
if
(
e
)
{
...
...
@@ -1871,7 +1856,7 @@ export class TextModel extends Disposable implements model.ITextModel {
const
eventBuilder
=
new
ModelTokensChangedEventBuilder
();
this
.
_tokens
.
_
updateTokensUntilLine
(
this
.
_buffer
,
eventBuilder
,
lineNumber
);
this
.
_tokens
.
updateTokensUntilLine
(
this
.
_buffer
,
eventBuilder
,
lineNumber
);
const
e
=
eventBuilder
.
build
();
if
(
e
)
{
...
...
@@ -1989,7 +1974,7 @@ export class TextModel extends Disposable implements model.ITextModel {
break
;
}
const
tokenizedLineNumber
=
this
.
_tokens
.
_tokenizeOne
Line
(
this
.
_buffer
,
eventBuilder
);
const
tokenizedLineNumber
=
this
.
_tokens
.
tokenizeOneInvalid
Line
(
this
.
_buffer
,
eventBuilder
);
if
(
tokenizedLineNumber
>=
toLineNumber
)
{
break
;
...
...
src/vs/editor/common/model/textModelTokens.ts
浏览文件 @
8a5f705e
...
...
@@ -388,7 +388,27 @@ class TokensStore {
//#endregion
}
export
class
ModelLinesTokens
{
export
interface
IModelLinesTokens
{
readonly
tokenizationSupport
:
ITokenizationSupport
|
null
;
readonly
invalidLineStartIndex
:
number
;
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
;
getTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineText
:
string
):
LineTokens
;
isCheapToTokenize
(
lineNumber
:
number
):
boolean
;
hasLinesToTokenize
(
buffer
:
ITextBuffer
):
boolean
;
getState
(
lineIndex
:
number
):
IState
|
null
;
applyEdits
(
range
:
Range
,
eolCount
:
number
,
firstLineLength
:
number
):
void
;
tokenizeOneInvalidLine
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
):
number
;
updateTokensUntilLine
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
lineNumber
:
number
):
void
;
fakeTokenizeLines
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
initialState
:
IState
,
startLineNumber
:
number
,
endLineNumber
:
number
):
void
;
_getAllStates
(
linesLength
:
number
):
(
IState
|
null
)[];
_getAllInvalid
(
linesLength
:
number
):
number
[];
}
export
class
ModelLinesTokens
implements
IModelLinesTokens
{
public
readonly
languageIdentifier
:
LanguageIdentifier
;
public
readonly
tokenizationSupport
:
ITokenizationSupport
|
null
;
...
...
@@ -427,19 +447,15 @@ export class ModelLinesTokens {
return
(
this
.
store
.
invalidLineStartIndex
<
buffer
.
getLineCount
());
}
_isInvalid
(
lineIndex
:
number
):
boolean
{
return
this
.
store
.
isInvalid
(
lineIndex
);
}
_getState
(
lineIndex
:
number
):
IState
|
null
{
public
getState
(
lineIndex
:
number
):
IState
|
null
{
return
this
.
store
.
getState
(
lineIndex
);
}
_
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
public
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
this
.
store
.
setTokens
(
topLevelLanguageId
,
lineIndex
,
lineTextLength
,
tokens
);
}
_setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
private
_setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
this
.
store
.
setState
(
lineIndex
,
state
);
}
...
...
@@ -449,7 +465,7 @@ export class ModelLinesTokens {
this
.
store
.
applyEdits
(
range
,
eolCount
,
firstLineLength
);
}
_invalidateLine
(
lineIndex
:
number
):
void
{
private
_invalidateLine
(
lineIndex
:
number
):
void
{
this
.
store
.
invalidateLine
(
lineIndex
);
}
...
...
@@ -457,33 +473,16 @@ export class ModelLinesTokens {
//#region Tokenization
public
_tokenizeOne
Line
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
):
number
{
public
tokenizeOneInvalid
Line
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
):
number
{
if
(
!
this
.
hasLinesToTokenize
(
buffer
))
{
return
buffer
.
getLineCount
()
+
1
;
}
const
lineNumber
=
this
.
store
.
invalidLineStartIndex
+
1
;
this
.
_
updateTokensUntilLine
(
buffer
,
eventBuilder
,
lineNumber
);
this
.
updateTokensUntilLine
(
buffer
,
eventBuilder
,
lineNumber
);
return
lineNumber
;
}
public
_tokenizeText
(
buffer
:
ITextBuffer
,
text
:
string
,
state
:
IState
):
TokenizationResult2
{
let
r
:
TokenizationResult2
|
null
=
null
;
if
(
this
.
tokenizationSupport
)
{
try
{
r
=
this
.
tokenizationSupport
.
tokenize2
(
text
,
state
,
0
);
}
catch
(
e
)
{
onUnexpectedError
(
e
);
}
}
if
(
!
r
)
{
r
=
nullTokenize2
(
this
.
languageIdentifier
.
id
,
text
,
state
,
0
);
}
return
r
;
}
public
_updateTokensUntilLine
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
lineNumber
:
number
):
void
{
public
updateTokensUntilLine
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
lineNumber
:
number
):
void
{
if
(
!
this
.
tokenizationSupport
)
{
this
.
store
.
_invalidLineStartIndex
=
buffer
.
getLineCount
();
return
;
...
...
@@ -495,28 +494,76 @@ export class ModelLinesTokens {
// Validate all states up to and including endLineIndex
for
(
let
lineIndex
=
this
.
store
.
invalidLineStartIndex
;
lineIndex
<=
endLineIndex
;
lineIndex
++
)
{
const
text
=
buffer
.
getLineContent
(
lineIndex
+
1
);
const
lineStartState
=
this
.
_getState
(
lineIndex
);
let
r
:
TokenizationResult2
|
null
=
null
;
try
{
// Tokenize only the first X characters
let
freshState
=
lineStartState
!
.
clone
();
r
=
this
.
tokenizationSupport
.
tokenize2
(
text
,
freshState
,
0
);
}
catch
(
e
)
{
onUnexpectedError
(
e
);
}
const
lineStartState
=
this
.
getState
(
lineIndex
);
if
(
!
r
)
{
r
=
nullTokenize2
(
this
.
languageIdentifier
.
id
,
text
,
lineStartState
,
0
);
}
const
r
=
safeTokenize
(
this
.
languageIdentifier
,
this
.
tokenizationSupport
,
text
,
lineStartState
!
);
this
.
store
.
setGoodTokens
(
this
.
languageIdentifier
.
id
,
linesLength
,
lineIndex
,
text
,
r
);
eventBuilder
.
registerChangedTokens
(
lineIndex
+
1
);
lineIndex
=
this
.
store
.
invalidLineStartIndex
-
1
;
// -1 because the outer loop increments it
}
}
public
fakeTokenizeLines
(
buffer
:
ITextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
initialState
:
IState
,
startLineNumber
:
number
,
endLineNumber
:
number
):
void
{
if
(
!
this
.
tokenizationSupport
)
{
return
;
}
let
state
=
initialState
;
for
(
let
i
=
startLineNumber
;
i
<=
endLineNumber
;
i
++
)
{
let
text
=
buffer
.
getLineContent
(
i
);
let
r
=
safeTokenize
(
this
.
languageIdentifier
,
this
.
tokenizationSupport
,
text
,
state
);
if
(
r
)
{
this
.
setTokens
(
this
.
languageIdentifier
.
id
,
i
-
1
,
text
.
length
,
r
.
tokens
);
// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
this
.
_invalidateLine
(
i
-
1
);
this
.
_setState
(
i
-
1
,
state
);
state
=
r
.
endState
;
eventBuilder
.
registerChangedTokens
(
i
);
}
else
{
state
=
initialState
;
}
}
}
// #endregion
_getAllStates
(
linesLength
:
number
):
(
IState
|
null
)[]
{
const
r
:
(
IState
|
null
)[]
=
[];
for
(
let
i
=
0
;
i
<
linesLength
;
i
++
)
{
r
[
i
]
=
this
.
getState
(
i
);
}
r
[
linesLength
]
=
this
.
store
.
_lastState
;
return
r
;
}
_getAllInvalid
(
linesLength
:
number
):
number
[]
{
const
r
:
number
[]
=
[];
for
(
let
i
=
0
;
i
<
linesLength
;
i
++
)
{
if
(
this
.
store
.
isInvalid
(
i
))
{
r
.
push
(
i
);
}
}
return
r
;
}
}
export
function
safeTokenize
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
|
null
,
text
:
string
,
state
:
IState
):
TokenizationResult2
{
let
r
:
TokenizationResult2
|
null
=
null
;
if
(
tokenizationSupport
)
{
try
{
r
=
tokenizationSupport
.
tokenize2
(
text
,
state
.
clone
(),
0
);
}
catch
(
e
)
{
onUnexpectedError
(
e
);
}
}
if
(
!
r
)
{
r
=
nullTokenize2
(
languageIdentifier
.
id
,
text
,
state
,
0
);
}
return
r
;
}
export
class
ModelTokensChangedEventBuilder
{
...
...
src/vs/editor/test/common/model/model.line.test.ts
浏览文件 @
8a5f705e
...
...
@@ -110,7 +110,7 @@ suite('ModelLinesTokens', () => {
for
(
let
lineIndex
=
0
;
lineIndex
<
initial
.
length
;
lineIndex
++
)
{
const
lineTokens
=
initial
[
lineIndex
].
tokens
;
const
lineTextLength
=
model
.
getLineMaxColumn
(
lineIndex
+
1
)
-
1
;
model
.
_tokens
.
_
setTokens
(
0
,
lineIndex
,
lineTextLength
,
TestToken
.
toTokens
(
lineTokens
));
model
.
_tokens
.
setTokens
(
0
,
lineIndex
,
lineTextLength
,
TestToken
.
toTokens
(
lineTokens
));
}
model
.
applyEdits
(
edits
.
map
((
ed
)
=>
({
...
...
@@ -441,14 +441,14 @@ suite('ModelLinesTokens', () => {
test
(
'
insertion on empty line
'
,
()
=>
{
const
model
=
new
TextModel
(
'
some text
'
,
TextModel
.
DEFAULT_CREATION_OPTIONS
,
new
LanguageIdentifier
(
'
test
'
,
0
));
model
.
_tokens
.
_
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
TestToken
.
toTokens
([
new
TestToken
(
0
,
1
)]));
model
.
_tokens
.
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
TestToken
.
toTokens
([
new
TestToken
(
0
,
1
)]));
model
.
applyEdits
([{
range
:
new
Range
(
1
,
1
,
1
,
10
),
text
:
''
}]);
model
.
_tokens
.
_
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
new
Uint32Array
(
0
));
model
.
_tokens
.
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
new
Uint32Array
(
0
));
model
.
applyEdits
([{
range
:
new
Range
(
1
,
1
,
1
,
1
),
...
...
src/vs/editor/test/common/model/model.modes.test.ts
浏览文件 @
8a5f705e
...
...
@@ -180,25 +180,18 @@ suite('Editor Model - Model Modes 2', () => {
};
function
invalidEqual
(
model
:
TextModel
,
expected
:
number
[]):
void
{
let
actual
:
number
[]
=
[];
for
(
let
i
=
0
,
len
=
model
.
getLineCount
();
i
<
len
;
i
++
)
{
if
(
model
.
_tokens
.
_isInvalid
(
i
))
{
actual
.
push
(
i
);
}
}
assert
.
deepEqual
(
actual
,
expected
);
assert
.
deepEqual
(
model
.
_tokens
.
_getAllInvalid
(
model
.
getLineCount
()),
expected
);
}
function
stateEqual
(
state
:
modes
.
IState
,
content
:
string
):
void
{
assert
.
equal
((
<
ModelState2
>
state
).
prevLineContent
,
content
);
}
function
statesEqual
(
model
:
TextModel
,
s
tates
:
string
[]):
void
{
let
i
,
len
=
states
.
length
-
1
;
for
(
i
=
0
;
i
<
len
;
i
++
)
{
stateEqual
(
model
.
_tokens
.
_getState
(
i
)
!
,
s
tates
[
i
]);
function
statesEqual
(
model
:
TextModel
,
expectedS
tates
:
string
[]):
void
{
const
actualStates
=
model
.
_tokens
.
_getAllStates
(
model
.
getLineCount
())
;
for
(
let
i
=
0
,
len
=
expectedStates
.
length
;
i
<
len
;
i
++
)
{
stateEqual
(
actualStates
[
i
]
!
,
expectedS
tates
[
i
]);
}
stateEqual
(
model
.
_tokens
.
store
.
_lastState
!
,
states
[
len
]);
}
let
thisModel
:
TextModel
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录