Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
xxadev
vscode
提交
3ee7787d
V
vscode
项目概览
xxadev
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3ee7787d
编写于
12月 22, 2017
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Move more tokenization related methods out of TextModelWithTokens
上级
d238ba5b
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
194 addition
and
211 deletion
+194
-211
src/vs/editor/common/model/modelLine.ts
src/vs/editor/common/model/modelLine.ts
+164
-11
src/vs/editor/common/model/textModelWithTokens.ts
src/vs/editor/common/model/textModelWithTokens.ts
+25
-195
src/vs/editor/test/common/model/model.line.test.ts
src/vs/editor/test/common/model/model.line.test.ts
+3
-3
src/vs/editor/test/common/model/model.modes.test.ts
src/vs/editor/test/common/model/model.modes.test.ts
+2
-2
未找到文件。
src/vs/editor/common/model/modelLine.ts
浏览文件 @
3ee7787d
...
...
@@ -4,12 +4,17 @@
*--------------------------------------------------------------------------------------------*/
'
use strict
'
;
import
{
IState
,
FontStyle
,
StandardTokenType
,
MetadataConsts
,
ColorId
,
LanguageId
}
from
'
vs/editor/common/modes
'
;
import
{
IState
,
FontStyle
,
StandardTokenType
,
MetadataConsts
,
ColorId
,
LanguageId
,
ITokenizationSupport
,
LanguageIdentifier
}
from
'
vs/editor/common/modes
'
;
import
{
CharCode
}
from
'
vs/base/common/charCode
'
;
import
{
LineTokens
}
from
'
vs/editor/common/core/lineTokens
'
;
import
*
as
arrays
from
'
vs/base/common/arrays
'
;
import
{
Position
}
from
'
vs/editor/common/core/position
'
;
import
{
Range
}
from
'
vs/editor/common/core/range
'
;
import
{
IModelTokensChangedEvent
}
from
'
vs/editor/common/model/textModelEvents
'
;
import
{
onUnexpectedError
}
from
'
vs/base/common/errors
'
;
import
{
TextBuffer
}
from
'
vs/editor/common/model/textBuffer
'
;
import
{
TokenizationResult2
}
from
'
vs/editor/common/core/token
'
;
import
{
nullTokenize2
}
from
'
vs/editor/common/modes/nullMode
'
;
export
interface
ILineEdit
{
startColumn
:
number
;
...
...
@@ -199,14 +204,32 @@ class ModelLineTokens {
export
class
ModelLinesTokens
{
public
readonly
languageIdentifier
:
LanguageIdentifier
;
public
readonly
tokenizationSupport
:
ITokenizationSupport
;
private
_tokens
:
ModelLineTokens
[];
private
_invalidLineStartIndex
:
number
;
private
_lastState
:
IState
;
constructor
()
{
constructor
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
)
{
this
.
languageIdentifier
=
languageIdentifier
;
this
.
tokenizationSupport
=
tokenizationSupport
;
this
.
_tokens
=
[];
}
if
(
this
.
tokenizationSupport
)
{
let
initialState
:
IState
=
null
;
try
{
initialState
=
this
.
tokenizationSupport
.
getInitialState
();
}
catch
(
e
)
{
onUnexpectedError
(
e
);
this
.
tokenizationSupport
=
null
;
}
if
(
initialState
)
{
this
.
_tokens
[
0
]
=
new
ModelLineTokens
(
initialState
);
}
}
public
setInitialState
(
initialState
:
IState
):
void
{
this
.
_
tokens
[
0
]
=
new
ModelLineTokens
(
initialState
)
;
this
.
_invalidLineStartIndex
=
0
;
this
.
_
lastState
=
null
;
}
public
getTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineText
:
string
):
LineTokens
{
...
...
@@ -225,27 +248,46 @@ export class ModelLinesTokens {
return
new
LineTokens
(
lineTokens
,
lineText
);
}
public
setIsInvalid
(
lineIndex
:
number
,
invalid
:
boolean
):
void
{
public
isCheapToTokenize
(
lineNumber
:
number
):
boolean
{
const
firstInvalidLineNumber
=
this
.
_invalidLineStartIndex
+
1
;
return
(
firstInvalidLineNumber
>=
lineNumber
);
}
public
hasLinesToTokenize
(
buffer
:
TextBuffer
):
boolean
{
return
(
this
.
_invalidLineStartIndex
<
buffer
.
getLineCount
());
}
public
invalidateLine
(
buffer
:
TextBuffer
,
lineIndex
:
number
):
void
{
this
.
_setIsInvalid
(
lineIndex
,
true
);
if
(
lineIndex
<
this
.
_invalidLineStartIndex
)
{
if
(
this
.
_invalidLineStartIndex
<
buffer
.
getLineCount
())
{
this
.
_setIsInvalid
(
this
.
_invalidLineStartIndex
,
true
);
}
this
.
_invalidLineStartIndex
=
lineIndex
;
}
}
private
_setIsInvalid
(
lineIndex
:
number
,
invalid
:
boolean
):
void
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
this
.
_tokens
[
lineIndex
].
_invalid
=
invalid
;
}
}
public
isInvalid
(
lineIndex
:
number
):
boolean
{
_
isInvalid
(
lineIndex
:
number
):
boolean
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
return
this
.
_tokens
[
lineIndex
].
_invalid
;
}
return
true
;
}
public
getState
(
lineIndex
:
number
):
IState
{
_
getState
(
lineIndex
:
number
):
IState
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
return
this
.
_tokens
[
lineIndex
].
_state
;
}
return
null
;
}
public
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
_
setTokens
(
topLevelLanguageId
:
LanguageId
,
lineIndex
:
number
,
lineTextLength
:
number
,
tokens
:
Uint32Array
):
void
{
let
target
:
ModelLineTokens
;
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
target
=
this
.
_tokens
[
lineIndex
];
...
...
@@ -270,7 +312,7 @@ export class ModelLinesTokens {
target
.
_lineTokens
=
tokens
.
buffer
;
}
p
ublic
setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
p
rivate
_
setState
(
lineIndex
:
number
,
state
:
IState
):
void
{
if
(
lineIndex
<
this
.
_tokens
.
length
)
{
this
.
_tokens
[
lineIndex
].
_state
=
state
;
}
else
{
...
...
@@ -279,7 +321,7 @@ export class ModelLinesTokens {
}
}
//
--- e
diting
//
#region E
diting
public
applyEdits
(
range
:
Range
,
lines
:
string
[]):
void
{
this
.
_acceptDeleteRange
(
range
);
...
...
@@ -349,4 +391,115 @@ export class ModelLinesTokens {
}
this
.
_tokens
=
arrays
.
arrayInsert
(
this
.
_tokens
,
position
.
lineNumber
,
insert
);
}
//#endregion
//#region Tokenization
public
_tokenizeOneLine
(
buffer
:
TextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
):
number
{
if
(
!
this
.
hasLinesToTokenize
(
buffer
))
{
return
buffer
.
getLineCount
()
+
1
;
}
const
lineNumber
=
this
.
_invalidLineStartIndex
+
1
;
this
.
_updateTokensUntilLine
(
buffer
,
eventBuilder
,
lineNumber
);
return
lineNumber
;
}
public
_updateTokensUntilLine
(
buffer
:
TextBuffer
,
eventBuilder
:
ModelTokensChangedEventBuilder
,
lineNumber
:
number
):
void
{
if
(
!
this
.
tokenizationSupport
)
{
this
.
_invalidLineStartIndex
=
buffer
.
getLineCount
();
return
;
}
const
linesLength
=
buffer
.
getLineCount
();
const
endLineIndex
=
lineNumber
-
1
;
// Validate all states up to and including endLineIndex
for
(
let
lineIndex
=
this
.
_invalidLineStartIndex
;
lineIndex
<=
endLineIndex
;
lineIndex
++
)
{
const
endStateIndex
=
lineIndex
+
1
;
let
r
:
TokenizationResult2
=
null
;
const
text
=
buffer
.
getLineContent
(
lineIndex
+
1
);
try
{
// Tokenize only the first X characters
let
freshState
=
this
.
_getState
(
lineIndex
).
clone
();
r
=
this
.
tokenizationSupport
.
tokenize2
(
text
,
freshState
,
0
);
}
catch
(
e
)
{
onUnexpectedError
(
e
);
}
if
(
!
r
)
{
r
=
nullTokenize2
(
this
.
languageIdentifier
.
id
,
text
,
this
.
_getState
(
lineIndex
),
0
);
}
this
.
_setTokens
(
this
.
languageIdentifier
.
id
,
lineIndex
,
text
.
length
,
r
.
tokens
);
eventBuilder
.
registerChangedTokens
(
lineIndex
+
1
);
this
.
_setIsInvalid
(
lineIndex
,
false
);
if
(
endStateIndex
<
linesLength
)
{
if
(
this
.
_getState
(
endStateIndex
)
!==
null
&&
r
.
endState
.
equals
(
this
.
_getState
(
endStateIndex
)))
{
// The end state of this line remains the same
let
nextInvalidLineIndex
=
lineIndex
+
1
;
while
(
nextInvalidLineIndex
<
linesLength
)
{
if
(
this
.
_isInvalid
(
nextInvalidLineIndex
))
{
break
;
}
if
(
nextInvalidLineIndex
+
1
<
linesLength
)
{
if
(
this
.
_getState
(
nextInvalidLineIndex
+
1
)
===
null
)
{
break
;
}
}
else
{
if
(
this
.
_lastState
===
null
)
{
break
;
}
}
nextInvalidLineIndex
++
;
}
this
.
_invalidLineStartIndex
=
Math
.
max
(
this
.
_invalidLineStartIndex
,
nextInvalidLineIndex
);
lineIndex
=
nextInvalidLineIndex
-
1
;
// -1 because the outer loop increments it
}
else
{
this
.
_setState
(
endStateIndex
,
r
.
endState
);
}
}
else
{
this
.
_lastState
=
r
.
endState
;
}
}
this
.
_invalidLineStartIndex
=
Math
.
max
(
this
.
_invalidLineStartIndex
,
endLineIndex
+
1
);
}
// #endregion
}
export
class
ModelTokensChangedEventBuilder
{
private
_ranges
:
{
fromLineNumber
:
number
;
toLineNumber
:
number
;
}[];
constructor
()
{
this
.
_ranges
=
[];
}
public
registerChangedTokens
(
lineNumber
:
number
):
void
{
const
ranges
=
this
.
_ranges
;
const
rangesLength
=
ranges
.
length
;
const
previousRange
=
rangesLength
>
0
?
ranges
[
rangesLength
-
1
]
:
null
;
if
(
previousRange
&&
previousRange
.
toLineNumber
===
lineNumber
-
1
)
{
// extend previous range
previousRange
.
toLineNumber
++
;
}
else
{
// insert new range
ranges
[
rangesLength
]
=
{
fromLineNumber
:
lineNumber
,
toLineNumber
:
lineNumber
};
}
}
public
build
():
IModelTokensChangedEvent
{
if
(
this
.
_ranges
.
length
===
0
)
{
return
null
;
}
return
{
ranges
:
this
.
_ranges
};
}
}
src/vs/editor/common/model/textModelWithTokens.ts
浏览文件 @
3ee7787d
...
...
@@ -4,66 +4,26 @@
*--------------------------------------------------------------------------------------------*/
'
use strict
'
;
import
*
as
nls
from
'
vs/nls
'
;
import
{
onUnexpectedError
}
from
'
vs/base/common/errors
'
;
import
{
IDisposable
}
from
'
vs/base/common/lifecycle
'
;
import
{
StopWatch
}
from
'
vs/base/common/stopwatch
'
;
import
Event
,
{
Emitter
}
from
'
vs/base/common/event
'
;
import
{
Range
}
from
'
vs/editor/common/core/range
'
;
import
*
as
editorCommon
from
'
vs/editor/common/editorCommon
'
;
import
{
TextModel
}
from
'
vs/editor/common/model/textModel
'
;
import
{
ITokenizationSupport
,
IState
,
TokenizationRegistry
,
LanguageId
,
LanguageIdentifier
}
from
'
vs/editor/common/modes
'
;
import
{
NULL_LANGUAGE_IDENTIFIER
,
nullTokenize2
}
from
'
vs/editor/common/modes/nullMode
'
;
import
{
TokenizationRegistry
,
LanguageId
,
LanguageIdentifier
}
from
'
vs/editor/common/modes
'
;
import
{
NULL_LANGUAGE_IDENTIFIER
}
from
'
vs/editor/common/modes/nullMode
'
;
import
{
ignoreBracketsInToken
}
from
'
vs/editor/common/modes/supports
'
;
import
{
BracketsUtils
,
RichEditBrackets
,
RichEditBracket
}
from
'
vs/editor/common/modes/supports/richEditBrackets
'
;
import
{
Position
,
IPosition
}
from
'
vs/editor/common/core/position
'
;
import
{
LanguageConfigurationRegistry
}
from
'
vs/editor/common/modes/languageConfigurationRegistry
'
;
import
{
LineTokens
}
from
'
vs/editor/common/core/lineTokens
'
;
import
{
getWordAtText
}
from
'
vs/editor/common/model/wordHelper
'
;
import
{
TokenizationResult2
}
from
'
vs/editor/common/core/token
'
;
import
{
ITextSource
,
IRawTextSource
}
from
'
vs/editor/common/model/textSource
'
;
import
{
IModelTokensChangedEvent
,
IModelLanguageChangedEvent
,
IModelLanguageConfigurationChangedEvent
}
from
'
vs/editor/common/model/textModelEvents
'
;
import
{
ModelLinesTokens
,
computeIndentLevel
}
from
'
vs/editor/common/model/modelLine
'
;
class
ModelTokensChangedEventBuilder
{
private
_ranges
:
{
fromLineNumber
:
number
;
toLineNumber
:
number
;
}[];
constructor
()
{
this
.
_ranges
=
[];
}
public
registerChangedTokens
(
lineNumber
:
number
):
void
{
const
ranges
=
this
.
_ranges
;
const
rangesLength
=
ranges
.
length
;
const
previousRange
=
rangesLength
>
0
?
ranges
[
rangesLength
-
1
]
:
null
;
if
(
previousRange
&&
previousRange
.
toLineNumber
===
lineNumber
-
1
)
{
// extend previous range
previousRange
.
toLineNumber
++
;
}
else
{
// insert new range
ranges
[
rangesLength
]
=
{
fromLineNumber
:
lineNumber
,
toLineNumber
:
lineNumber
};
}
}
public
build
():
IModelTokensChangedEvent
{
if
(
this
.
_ranges
.
length
===
0
)
{
return
null
;
}
return
{
ranges
:
this
.
_ranges
};
}
}
import
{
ModelLinesTokens
,
computeIndentLevel
,
ModelTokensChangedEventBuilder
}
from
'
vs/editor/common/model/modelLine
'
;
export
class
TextModelWithTokens
extends
TextModel
implements
editorCommon
.
ITokenizedModel
{
private
static
readonly
MODE_TOKENIZATION_FAILED_MSG
=
nls
.
localize
(
'
mode.tokenizationSupportFailed
'
,
"
The mode has failed while tokenizing the input.
"
);
private
readonly
_onDidChangeLanguage
:
Emitter
<
IModelLanguageChangedEvent
>
=
this
.
_register
(
new
Emitter
<
IModelLanguageChangedEvent
>
());
public
readonly
onDidChangeLanguage
:
Event
<
IModelLanguageChangedEvent
>
=
this
.
_onDidChangeLanguage
.
event
;
...
...
@@ -75,10 +35,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
private
_languageIdentifier
:
LanguageIdentifier
;
private
_tokenizationListener
:
IDisposable
;
private
_tokenizationSupport
:
ITokenizationSupport
;
private
_invalidLineStartIndex
:
number
;
private
_lastState
:
IState
;
private
_languageRegistryListener
:
IDisposable
;
...
...
@@ -122,7 +78,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
this
.
_tokenizationListener
.
dispose
();
this
.
_languageRegistryListener
.
dispose
();
this
.
_clearTimers
();
this
.
_
lastState
=
null
;
this
.
_
tokens
=
null
;
super
.
dispose
();
}
...
...
@@ -139,30 +95,12 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
protected
_resetTokenizationState
():
void
{
this
.
_clearTimers
();
this
.
_tokens
=
new
ModelLinesTokens
();
this
.
_tokenizationSupport
=
null
;
if
(
!
this
.
_isTooLargeForTokenization
)
{
this
.
_tokenizationSupport
=
TokenizationRegistry
.
get
(
this
.
_languageIdentifier
.
language
);
}
if
(
this
.
_tokenizationSupport
)
{
let
initialState
:
IState
=
null
;
try
{
initialState
=
this
.
_tokenizationSupport
.
getInitialState
();
}
catch
(
e
)
{
e
.
friendlyMessage
=
TextModelWithTokens
.
MODE_TOKENIZATION_FAILED_MSG
;
onUnexpectedError
(
e
);
this
.
_tokenizationSupport
=
null
;
}
if
(
initialState
)
{
this
.
_tokens
.
setInitialState
(
initialState
);
}
}
this
.
_lastState
=
null
;
this
.
_invalidLineStartIndex
=
0
;
let
tokenizationSupport
=
(
this
.
_isTooLargeForTokenization
?
null
:
TokenizationRegistry
.
get
(
this
.
_languageIdentifier
.
language
)
);
this
.
_tokens
=
new
ModelLinesTokens
(
this
.
_languageIdentifier
,
tokenizationSupport
);
this
.
_beginBackgroundTokenization
();
}
...
...
@@ -180,7 +118,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
const
eventBuilder
=
new
ModelTokensChangedEventBuilder
();
this
.
_
updateTokensUntilLine
(
eventBuilder
,
lineNumber
);
this
.
_
tokens
.
_updateTokensUntilLine
(
this
.
_buffer
,
eventBuilder
,
lineNumber
);
const
e
=
eventBuilder
.
build
();
if
(
e
)
{
...
...
@@ -189,8 +127,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}
public
isCheapToTokenize
(
lineNumber
:
number
):
boolean
{
const
firstInvalidLineNumber
=
this
.
_invalidLineStartIndex
+
1
;
return
(
firstInvalidLineNumber
>=
lineNumber
);
return
this
.
_tokens
.
isCheapToTokenize
(
lineNumber
);
}
public
tokenizeIfCheap
(
lineNumber
:
number
):
void
{
...
...
@@ -247,7 +184,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}
public
getLanguageIdAtPosition
(
_lineNumber
:
number
,
_column
:
number
):
LanguageId
{
if
(
!
this
.
_tokenizationSupport
)
{
if
(
!
this
.
_token
s
.
token
izationSupport
)
{
return
this
.
_languageIdentifier
.
id
;
}
let
{
lineNumber
,
column
}
=
this
.
validatePosition
({
lineNumber
:
_lineNumber
,
column
:
_column
});
...
...
@@ -257,12 +194,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}
protected
_invalidateLine
(
lineIndex
:
number
):
void
{
this
.
_tokens
.
setIsInvalid
(
lineIndex
,
true
);
if
(
lineIndex
<
this
.
_invalidLineStartIndex
)
{
if
(
this
.
_invalidLineStartIndex
<
this
.
_buffer
.
getLineCount
())
{
this
.
_tokens
.
setIsInvalid
(
this
.
_invalidLineStartIndex
,
true
);
}
this
.
_invalidLineStartIndex
=
lineIndex
;
this
.
_tokens
.
invalidateLine
(
this
.
_buffer
,
lineIndex
);
if
(
this
.
_tokens
.
hasLinesToTokenize
(
this
.
_buffer
))
{
this
.
_beginBackgroundTokenization
();
}
}
...
...
@@ -281,60 +214,30 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
var
maxLineNumber
=
Math
.
min
(
100
,
this
.
getLineCount
());
this
.
_revalidateTokensNow
(
maxLineNumber
);
if
(
this
.
_
invalidLineStartIndex
<
this
.
_buffer
.
getLineCount
(
))
{
if
(
this
.
_
tokens
.
hasLinesToTokenize
(
this
.
_buffer
))
{
this
.
_beginBackgroundTokenization
();
}
}
private
_revalidateTokensNow
(
toLineNumber
:
number
=
this
.
_
invalidLineStartIndex
+
1000000
):
void
{
private
_revalidateTokensNow
(
toLineNumber
:
number
=
this
.
_
buffer
.
getLineCount
()
):
void
{
const
MAX_ALLOWED_TIME
=
20
;
const
eventBuilder
=
new
ModelTokensChangedEventBuilder
();
const
sw
=
StopWatch
.
create
(
false
);
toLineNumber
=
Math
.
min
(
this
.
_buffer
.
getLineCount
(),
toLineNumber
);
var
MAX_ALLOWED_TIME
=
20
,
fromLineNumber
=
this
.
_invalidLineStartIndex
+
1
,
tokenizedChars
=
0
,
currentCharsToTokenize
=
0
,
currentEstimatedTimeToTokenize
=
0
,
sw
=
StopWatch
.
create
(
false
),
elapsedTime
:
number
;
// Tokenize at most 1000 lines. Estimate the tokenization speed per character and stop when:
// - MAX_ALLOWED_TIME is reached
// - tokenizing the next line would go above MAX_ALLOWED_TIME
for
(
var
lineNumber
=
fromLineNumber
;
lineNumber
<=
toLineNumber
;
lineNumber
++
)
{
elapsedTime
=
sw
.
elapsed
();
if
(
elapsedTime
>
MAX_ALLOWED_TIME
)
{
while
(
this
.
_tokens
.
hasLinesToTokenize
(
this
.
_buffer
))
{
if
(
sw
.
elapsed
()
>
MAX_ALLOWED_TIME
)
{
// Stop if MAX_ALLOWED_TIME is reached
toLineNumber
=
lineNumber
-
1
;
break
;
}
// Compute how many characters will be tokenized for this line
currentCharsToTokenize
=
this
.
_buffer
.
getLineLength
(
lineNumber
);
const
tokenizedLineNumber
=
this
.
_tokens
.
_tokenizeOneLine
(
this
.
_buffer
,
eventBuilder
);
if
(
tokenizedChars
>
0
)
{
// If we have enough history, estimate how long tokenizing this line would take
currentEstimatedTimeToTokenize
=
(
elapsedTime
/
tokenizedChars
)
*
currentCharsToTokenize
;
if
(
elapsedTime
+
currentEstimatedTimeToTokenize
>
MAX_ALLOWED_TIME
)
{
// Tokenizing this line will go above MAX_ALLOWED_TIME
toLineNumber
=
lineNumber
-
1
;
break
;
}
if
(
tokenizedLineNumber
>=
toLineNumber
)
{
break
;
}
this
.
_updateTokensUntilLine
(
eventBuilder
,
lineNumber
);
tokenizedChars
+=
currentCharsToTokenize
;
// Skip the lines that got tokenized
lineNumber
=
Math
.
max
(
lineNumber
,
this
.
_invalidLineStartIndex
+
1
);
}
elapsedTime
=
sw
.
elapsed
();
if
(
this
.
_invalidLineStartIndex
<
this
.
_buffer
.
getLineCount
())
{
if
(
this
.
_tokens
.
hasLinesToTokenize
(
this
.
_buffer
))
{
this
.
_beginBackgroundTokenization
();
}
...
...
@@ -344,68 +247,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}
}
private
_updateTokensUntilLine
(
eventBuilder
:
ModelTokensChangedEventBuilder
,
lineNumber
:
number
):
void
{
if
(
!
this
.
_tokenizationSupport
)
{
this
.
_invalidLineStartIndex
=
this
.
_buffer
.
getLineCount
();
return
;
}
const
linesLength
=
this
.
_buffer
.
getLineCount
();
const
endLineIndex
=
lineNumber
-
1
;
// Validate all states up to and including endLineIndex
for
(
let
lineIndex
=
this
.
_invalidLineStartIndex
;
lineIndex
<=
endLineIndex
;
lineIndex
++
)
{
const
endStateIndex
=
lineIndex
+
1
;
let
r
:
TokenizationResult2
=
null
;
const
text
=
this
.
_buffer
.
getLineContent
(
lineIndex
+
1
);
try
{
// Tokenize only the first X characters
let
freshState
=
this
.
_tokens
.
getState
(
lineIndex
).
clone
();
r
=
this
.
_tokenizationSupport
.
tokenize2
(
text
,
freshState
,
0
);
}
catch
(
e
)
{
e
.
friendlyMessage
=
TextModelWithTokens
.
MODE_TOKENIZATION_FAILED_MSG
;
onUnexpectedError
(
e
);
}
if
(
!
r
)
{
r
=
nullTokenize2
(
this
.
_languageIdentifier
.
id
,
text
,
this
.
_tokens
.
getState
(
lineIndex
),
0
);
}
this
.
_tokens
.
setTokens
(
this
.
_languageIdentifier
.
id
,
lineIndex
,
text
.
length
,
r
.
tokens
);
eventBuilder
.
registerChangedTokens
(
lineIndex
+
1
);
this
.
_tokens
.
setIsInvalid
(
lineIndex
,
false
);
if
(
endStateIndex
<
linesLength
)
{
if
(
this
.
_tokens
.
getState
(
endStateIndex
)
!==
null
&&
r
.
endState
.
equals
(
this
.
_tokens
.
getState
(
endStateIndex
)))
{
// The end state of this line remains the same
let
nextInvalidLineIndex
=
lineIndex
+
1
;
while
(
nextInvalidLineIndex
<
linesLength
)
{
if
(
this
.
_tokens
.
isInvalid
(
nextInvalidLineIndex
))
{
break
;
}
if
(
nextInvalidLineIndex
+
1
<
linesLength
)
{
if
(
this
.
_tokens
.
getState
(
nextInvalidLineIndex
+
1
)
===
null
)
{
break
;
}
}
else
{
if
(
this
.
_lastState
===
null
)
{
break
;
}
}
nextInvalidLineIndex
++
;
}
this
.
_invalidLineStartIndex
=
Math
.
max
(
this
.
_invalidLineStartIndex
,
nextInvalidLineIndex
);
lineIndex
=
nextInvalidLineIndex
-
1
;
// -1 because the outer loop increments it
}
else
{
this
.
_tokens
.
setState
(
endStateIndex
,
r
.
endState
);
}
}
else
{
this
.
_lastState
=
r
.
endState
;
}
}
this
.
_invalidLineStartIndex
=
Math
.
max
(
this
.
_invalidLineStartIndex
,
endLineIndex
+
1
);
}
private
emitModelTokensChangedEvent
(
e
:
IModelTokensChangedEvent
):
void
{
if
(
!
this
.
_isDisposing
)
{
this
.
_onDidChangeTokens
.
fire
(
e
);
...
...
@@ -418,17 +259,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
this
.
_assertNotDisposed
();
const
position
=
this
.
validatePosition
(
_position
);
const
lineContent
=
this
.
getLineContent
(
position
.
lineNumber
);
if
(
this
.
_invalidLineStartIndex
<=
position
.
lineNumber
-
1
)
{
// this line is not tokenized
return
getWordAtText
(
position
.
column
,
LanguageConfigurationRegistry
.
getWordDefinition
(
this
.
_languageIdentifier
.
id
),
lineContent
,
0
);
}
const
lineTokens
=
this
.
_getLineTokens
(
position
.
lineNumber
);
const
offset
=
position
.
column
-
1
;
const
tokenIndex
=
lineTokens
.
findTokenIndexAtOffset
(
offset
);
...
...
src/vs/editor/test/common/model/model.line.test.ts
浏览文件 @
3ee7787d
...
...
@@ -289,7 +289,7 @@ suite('ModelLinesTokens', () => {
for
(
let
lineIndex
=
0
;
lineIndex
<
initial
.
length
;
lineIndex
++
)
{
const
lineTokens
=
initial
[
lineIndex
].
tokens
;
const
lineTextLength
=
model
.
getLineMaxColumn
(
lineIndex
+
1
)
-
1
;
model
.
_tokens
.
setTokens
(
0
,
lineIndex
,
lineTextLength
,
TestToken
.
toTokens
(
lineTokens
));
model
.
_tokens
.
_
setTokens
(
0
,
lineIndex
,
lineTextLength
,
TestToken
.
toTokens
(
lineTokens
));
}
model
.
applyEdits
(
edits
.
map
((
ed
)
=>
({
...
...
@@ -620,7 +620,7 @@ suite('ModelLinesTokens', () => {
test
(
'
insertion on empty line
'
,
()
=>
{
const
model
=
new
EditableTextModel
(
RawTextSource
.
fromString
(
'
some text
'
),
TextModel
.
DEFAULT_CREATION_OPTIONS
,
new
LanguageIdentifier
(
'
test
'
,
0
));
model
.
_tokens
.
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
TestToken
.
toTokens
([
new
TestToken
(
0
,
1
)]));
model
.
_tokens
.
_
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
TestToken
.
toTokens
([
new
TestToken
(
0
,
1
)]));
model
.
applyEdits
([{
identifier
:
null
,
...
...
@@ -629,7 +629,7 @@ suite('ModelLinesTokens', () => {
forceMoveMarkers
:
false
}]);
model
.
_tokens
.
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
new
Uint32Array
(
0
));
model
.
_tokens
.
_
setTokens
(
0
,
0
,
model
.
getLineMaxColumn
(
1
)
-
1
,
new
Uint32Array
(
0
));
model
.
applyEdits
([{
identifier
:
null
,
...
...
src/vs/editor/test/common/model/model.modes.test.ts
浏览文件 @
3ee7787d
...
...
@@ -185,7 +185,7 @@ suite('Editor Model - Model Modes 2', () => {
function
invalidEqual
(
model
:
Model
,
expected
:
number
[]):
void
{
let
actual
:
number
[]
=
[];
for
(
let
i
=
0
,
len
=
model
.
getLineCount
();
i
<
len
;
i
++
)
{
if
(
model
.
_tokens
.
isInvalid
(
i
))
{
if
(
model
.
_tokens
.
_
isInvalid
(
i
))
{
actual
.
push
(
i
);
}
}
...
...
@@ -199,7 +199,7 @@ suite('Editor Model - Model Modes 2', () => {
function
statesEqual
(
model
:
Model
,
states
:
string
[]):
void
{
var
i
,
len
=
states
.
length
-
1
;
for
(
i
=
0
;
i
<
len
;
i
++
)
{
stateEqual
(
model
.
_tokens
.
getState
(
i
),
states
[
i
]);
stateEqual
(
model
.
_tokens
.
_
getState
(
i
),
states
[
i
]);
}
stateEqual
((
<
any
>
model
).
_lastState
,
states
[
len
]);
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录