Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
xxadev
vscode
提交
921fde65
V
vscode
项目概览
xxadev
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
921fde65
编写于
12月 20, 2016
作者:
A
Alex Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Reduce usage of TokenizationSupport
上级
bec4fd8d
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
111 addition
and
164 deletion
+111
-164
src/vs/editor/common/model/textModelWithTokens.ts
src/vs/editor/common/model/textModelWithTokens.ts
+2
-1
src/vs/editor/common/modes.ts
src/vs/editor/common/modes.ts
+0
-5
src/vs/editor/common/modes/supports.ts
src/vs/editor/common/modes/supports.ts
+0
-3
src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts
...rib/smartSelect/test/common/tokenSelectionSupport.test.ts
+2
-2
src/vs/editor/test/common/mocks/mockMode.ts
src/vs/editor/test/common/mocks/mockMode.ts
+1
-38
src/vs/editor/test/common/model/model.modes.test.ts
src/vs/editor/test/common/model/model.modes.test.ts
+73
-82
src/vs/editor/test/common/model/textModelWithTokens.test.ts
src/vs/editor/test/common/model/textModelWithTokens.test.ts
+11
-5
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
+22
-24
src/vs/monaco.d.ts
src/vs/monaco.d.ts
+0
-4
未找到文件。
src/vs/editor/common/model/textModelWithTokens.ts
浏览文件 @
921fde65
...
...
@@ -365,7 +365,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
try
{
// Tokenize only the first X characters
r
=
this
.
_tokenizationSupport
.
tokenize
(
this
.
_lines
[
lineIndex
].
text
,
this
.
_lines
[
lineIndex
].
getState
(),
0
,
stopLineTokenizationAfter
);
let
freshState
=
this
.
_lines
[
lineIndex
].
getState
().
clone
();
r
=
this
.
_tokenizationSupport
.
tokenize
(
this
.
_lines
[
lineIndex
].
text
,
freshState
,
0
,
stopLineTokenizationAfter
);
}
catch
(
e
)
{
e
.
friendlyMessage
=
TextModelWithTokens
.
MODE_TOKENIZATION_FAILED_MSG
;
onUnexpectedError
(
e
);
...
...
src/vs/editor/common/modes.ts
浏览文件 @
921fde65
...
...
@@ -7,7 +7,6 @@
import
{
MarkedString
}
from
'
vs/base/common/htmlContent
'
;
import
{
IDisposable
}
from
'
vs/base/common/lifecycle
'
;
import
URI
from
'
vs/base/common/uri
'
;
import
{
TPromise
}
from
'
vs/base/common/winjs.base
'
;
import
{
IFilter
}
from
'
vs/base/common/filters
'
;
import
*
as
editorCommon
from
'
vs/editor/common/editorCommon
'
;
import
{
ModeTransition
}
from
'
vs/editor/common/core/modeTransition
'
;
...
...
@@ -87,10 +86,6 @@ export interface ILineTokens2 {
* A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned.
*/
endState
:
IState2
;
/**
* An optional promise to force the model to retokenize this line (e.g. missing information at the point of tokenization)
*/
retokenize
?:
TPromise
<
void
>
;
}
/**
* The state of the tokenizer between two lines.
...
...
src/vs/editor/common/modes/supports.ts
浏览文件 @
921fde65
...
...
@@ -4,7 +4,6 @@
*--------------------------------------------------------------------------------------------*/
'
use strict
'
;
import
{
TPromise
}
from
'
vs/base/common/winjs.base
'
;
import
*
as
modes
from
'
vs/editor/common/modes
'
;
import
{
ModeTransition
}
from
'
vs/editor/common/core/modeTransition
'
;
import
{
Token
}
from
'
vs/editor/common/core/token
'
;
...
...
@@ -17,14 +16,12 @@ export class RawLineTokens implements modes.ILineTokens {
modeTransitions
:
ModeTransition
[];
actualStopOffset
:
number
;
endState
:
modes
.
IState
;
retokenize
:
TPromise
<
void
>
;
constructor
(
tokens
:
Token
[],
modeTransitions
:
ModeTransition
[],
actualStopOffset
:
number
,
endState
:
modes
.
IState
)
{
this
.
tokens
=
tokens
;
this
.
modeTransitions
=
modeTransitions
;
this
.
actualStopOffset
=
actualStopOffset
;
this
.
endState
=
endState
;
this
.
retokenize
=
null
;
}
}
...
...
src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts
浏览文件 @
921fde65
...
...
@@ -10,12 +10,12 @@ import { Range } from 'vs/editor/common/core/range';
import
{
IMode
}
from
'
vs/editor/common/modes
'
;
import
{
IndentAction
}
from
'
vs/editor/common/modes/languageConfiguration
'
;
import
{
TokenSelectionSupport
}
from
'
vs/editor/contrib/smartSelect/common/tokenSelectionSupport
'
;
import
{
Mock
Tokenizing
Mode
}
from
'
vs/editor/test/common/mocks/mockMode
'
;
import
{
MockMode
}
from
'
vs/editor/test/common/mocks/mockMode
'
;
import
{
LanguageConfigurationRegistry
}
from
'
vs/editor/common/modes/languageConfigurationRegistry
'
;
import
{
ModelServiceImpl
}
from
'
vs/editor/common/services/modelServiceImpl
'
;
import
{
TestConfigurationService
}
from
'
vs/platform/configuration/test/common/testConfigurationService
'
;
class
MockJSMode
extends
Mock
Tokenizing
Mode
{
class
MockJSMode
extends
MockMode
{
constructor
()
{
super
(
'
mock-js
'
);
...
...
src/vs/editor/test/common/mocks/mockMode.ts
浏览文件 @
921fde65
...
...
@@ -4,10 +4,7 @@
*--------------------------------------------------------------------------------------------*/
'
use strict
'
;
import
{
IMode
,
IState
,
TokenizationRegistry
}
from
'
vs/editor/common/modes
'
;
import
{
AbstractState
,
ITokenizationResult
}
from
'
vs/editor/common/modes/abstractState
'
;
import
{
TokenizationSupport
}
from
'
vs/editor/common/modes/supports/tokenizationSupport
'
;
import
{
LineStream
}
from
'
vs/editor/common/modes/lineStream
'
;
import
{
IMode
}
from
'
vs/editor/common/modes
'
;
let
instanceCount
=
0
;
function
generateMockModeId
():
string
{
...
...
@@ -28,37 +25,3 @@ export class MockMode implements IMode {
return
this
.
_id
;
}
}
export
class
StateForMockTokenizingMode
extends
AbstractState
{
private
_tokenType
:
string
;
constructor
(
modeId
:
string
,
tokenType
:
string
)
{
super
(
modeId
);
this
.
_tokenType
=
tokenType
;
}
public
makeClone
():
StateForMockTokenizingMode
{
return
this
;
}
public
equals
(
other
:
IState
):
boolean
{
return
true
;
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
stream
.
advanceToEOS
();
return
{
type
:
this
.
_tokenType
};
}
}
export
class
MockTokenizingMode
extends
MockMode
{
constructor
(
tokenType
:
string
)
{
super
();
TokenizationRegistry
.
register
(
this
.
getId
(),
new
TokenizationSupport
(
null
,
this
.
getId
(),
{
getInitialState
:
()
=>
new
StateForMockTokenizingMode
(
this
.
getId
(),
tokenType
)
},
false
));
}
}
src/vs/editor/test/common/model/model.modes.test.ts
浏览文件 @
921fde65
...
...
@@ -9,20 +9,11 @@ import { EditOperation } from 'vs/editor/common/core/editOperation';
import
{
Position
}
from
'
vs/editor/common/core/position
'
;
import
{
Range
}
from
'
vs/editor/common/core/range
'
;
import
{
Model
}
from
'
vs/editor/common/model/model
'
;
import
{
AbstractState
,
ITokenizationResult
}
from
'
vs/editor/common/modes/abstractState
'
;
import
*
as
modes
from
'
vs/editor/common/modes
'
;
import
{
TokenizationSupport
}
from
'
vs/editor/common/modes/supports/tokenizationSupport
'
;
import
{
LineStream
}
from
'
vs/editor/common/modes/lineStream
'
;
import
{
Token
}
from
'
vs/editor/common/core/token
'
;
// --------- utils
var
LINE1
=
'
1
'
;
var
LINE2
=
'
2
'
;
var
LINE3
=
'
3
'
;
var
LINE4
=
'
4
'
;
var
LINE5
=
'
5
'
;
suite
(
'
Editor Model - Model Modes 1
'
,
()
=>
{
const
LANGUAGE_ID
=
'
modelModeTest1
'
;
...
...
@@ -32,39 +23,40 @@ suite('Editor Model - Model Modes 1', () => {
};
let
thisModel
:
Model
;
class
ModelState1
extends
AbstractState
{
public
makeClone
():
ModelState1
{
return
this
;
}
public
equals
(
other
:
modes
.
IState
):
boolean
{
return
this
===
other
;
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
let
chr
=
stream
.
peek
();
stream
.
advance
(
1
);
calledState
.
calledFor
.
push
(
chr
);
stream
.
advanceToEOS
();
return
{
type
:
''
};
}
}
function
checkAndClear
(
calledState
:
{
calledFor
:
string
[]
},
arr
:
string
[])
{
assert
.
deepEqual
(
calledState
.
calledFor
,
arr
);
calledState
.
calledFor
=
[];
}
modes
.
TokenizationRegistry
.
register
(
LANGUAGE_ID
,
new
TokenizationSupport
(
null
,
LANGUAGE_ID
,
{
getInitialState
:
()
=>
new
ModelState1
(
LANGUAGE_ID
)
},
false
));
class
ModelState1
implements
modes
.
IState
{
clone
():
modes
.
IState
{
return
this
;
}
equals
(
other
:
modes
.
IState
):
boolean
{
return
this
===
other
;
}
getModeId
():
string
{
return
LANGUAGE_ID
;
}
getStateData
():
modes
.
IState
{
throw
new
Error
(
'
Not implemented
'
);
}
setStateData
(
state
:
modes
.
IState
):
void
{
throw
new
Error
(
'
Not implemented
'
);
}
}
modes
.
TokenizationRegistry
.
register
(
LANGUAGE_ID
,
{
getInitialState
:
()
=>
new
ModelState1
(),
tokenize
:
(
line
:
string
,
state
:
modes
.
IState
):
modes
.
ILineTokens
=>
{
calledState
.
calledFor
.
push
(
line
.
charAt
(
0
));
return
{
tokens
:
[
new
Token
(
0
,
''
)],
actualStopOffset
:
line
.
length
,
endState
:
state
,
modeTransitions
:
null
};
}
});
setup
(()
=>
{
calledState
.
calledFor
=
[];
var
text
=
LINE1
+
'
\r\n
'
+
LINE2
+
'
\n
'
+
LINE3
+
'
\n
'
+
LINE4
+
'
\r\n
'
+
LINE5
;
'
1
\r\n
'
+
'
2
\n
'
+
'
3
\n
'
+
'
4
\r\n
'
+
'
5
'
;
thisModel
=
Model
.
createFromString
(
text
,
undefined
,
LANGUAGE_ID
);
});
...
...
@@ -168,38 +160,45 @@ suite('Editor Model - Model Modes 1', () => {
});
});
suite
(
'
Editor Model - Model Modes 2
'
,
()
=>
{
const
LANGUAGE_ID
=
'
modelModeTest2
'
;
class
ModelState2
extends
AbstractState
{
private
prevLineContent
:
string
;
class
ModelState2
implements
modes
.
IState
{
prevLineContent
:
string
;
constructor
(
modeId
:
string
,
prevLineContent
:
string
)
{
super
(
modeId
);
constructor
(
prevLineContent
:
string
)
{
this
.
prevLineContent
=
prevLineContent
;
}
public
makeClone
():
ModelState2
{
return
new
ModelState2
(
this
.
getModeId
(),
this
.
prevLineContent
);
clone
():
modes
.
IState
{
return
new
ModelState2
(
this
.
prevLineContent
);
}
public
equals
(
other
:
modes
.
IState
):
boolean
{
return
(
other
instanceof
ModelState2
)
&&
(
this
.
prevLineContent
===
(
<
ModelState2
>
other
).
prevLineContent
)
;
equals
(
other
:
modes
.
IState
):
boolean
{
return
(
other
instanceof
ModelState2
)
&&
other
.
prevLineContent
===
this
.
prevLineContent
;
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
var
line
=
stream
.
advanceToEOS
();
this
.
prevLineContent
=
line
;
return
{
type
:
''
};
getModeId
():
string
{
return
LANGUAGE_ID
;
}
getStateData
():
modes
.
IState
{
throw
new
Error
(
'
Not implemented
'
);
}
setStateData
(
state
:
modes
.
IState
):
void
{
throw
new
Error
(
'
Not implemented
'
);
}
}
modes
.
TokenizationRegistry
.
register
(
LANGUAGE_ID
,
new
TokenizationSupport
(
null
,
LANGUAGE_ID
,
{
getInitialState
:
()
=>
new
ModelState2
(
LANGUAGE_ID
,
''
)
},
false
));
modes
.
TokenizationRegistry
.
register
(
LANGUAGE_ID
,
{
getInitialState
:
()
=>
new
ModelState2
(
''
),
tokenize
:
(
line
:
string
,
state
:
modes
.
IState
):
modes
.
ILineTokens
=>
{
(
<
ModelState2
>
state
).
prevLineContent
=
line
;
return
{
tokens
:
[
new
Token
(
0
,
''
)],
actualStopOffset
:
line
.
length
,
endState
:
state
,
modeTransitions
:
null
};
}
});
function
invalidEqual
(
model
,
indexArray
)
{
var
i
,
len
,
asHash
=
{};
...
...
@@ -306,39 +305,31 @@ suite('Editor Model - Token Iterator', () => {
const
LANGUAGE_ID
=
'
modelModeTestTokenIterator
'
;
class
NState
extends
AbstractState
{
private
n
:
number
;
private
allResults
:
ITokenizationResult
[];
constructor
(
modeId
:
string
,
n
:
number
)
{
super
(
modeId
);
this
.
n
=
n
;
this
.
allResults
=
null
;
}
public
makeClone
():
NState
{
return
this
;
}
public
equals
(
other
:
modes
.
IState
):
boolean
{
return
true
;
}
class
NState
implements
modes
.
IState
{
clone
():
modes
.
IState
{
return
this
;
}
equals
(
other
:
modes
.
IState
):
boolean
{
return
this
===
other
;
}
getModeId
():
string
{
return
LANGUAGE_ID
;
}
getStateData
():
modes
.
IState
{
throw
new
Error
(
'
Not implemented
'
);
}
setStateData
(
state
:
modes
.
IState
):
void
{
throw
new
Error
(
'
Not implemented
'
);
}
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
var
ndash
=
this
.
n
,
value
=
''
;
while
(
!
stream
.
eos
()
&&
ndash
>
0
)
{
let
chr
=
stream
.
peek
();
stream
.
advance
(
1
);
value
+=
chr
;
ndash
--
;
modes
.
TokenizationRegistry
.
register
(
LANGUAGE_ID
,
{
getInitialState
:
():
modes
.
IState
=>
new
NState
(),
tokenize
:
(
line
:
string
,
state
:
modes
.
IState
):
modes
.
ILineTokens
=>
{
let
tokens
:
Token
[]
=
[];
for
(
let
i
=
0
;
i
<
line
.
length
/
3
;
i
++
)
{
let
from
=
3
*
i
;
let
to
=
from
+
3
;
tokens
.
push
(
new
Token
(
from
,
'
n-3-
'
+
line
.
substring
(
from
,
to
)));
}
return
{
type
:
'
n-
'
+
(
this
.
n
-
ndash
)
+
'
-
'
+
value
};
return
{
tokens
:
tokens
,
actualStopOffset
:
line
.
length
,
endState
:
state
,
modeTransitions
:
null
};
}
}
modes
.
TokenizationRegistry
.
register
(
LANGUAGE_ID
,
new
TokenizationSupport
(
null
,
LANGUAGE_ID
,
{
getInitialState
:
()
=>
new
NState
(
LANGUAGE_ID
,
3
)
},
false
));
});
var
thisModel
:
Model
;
...
...
src/vs/editor/test/common/model/textModelWithTokens.test.ts
浏览文件 @
921fde65
...
...
@@ -7,7 +7,7 @@
import
*
as
assert
from
'
assert
'
;
import
{
Model
}
from
'
vs/editor/common/model/model
'
;
import
{
ViewLineToken
}
from
'
vs/editor/common/core/viewLineToken
'
;
import
{
TokenizationRegistry
}
from
'
vs/editor/common/modes
'
;
import
{
TokenizationRegistry
,
IState
}
from
'
vs/editor/common/modes
'
;
import
{
CharacterPair
}
from
'
vs/editor/common/modes/languageConfiguration
'
;
import
{
MockMode
}
from
'
vs/editor/test/common/mocks/mockMode
'
;
import
{
Token
}
from
'
vs/editor/common/core/token
'
;
...
...
@@ -260,21 +260,27 @@ suite('TextModelWithTokens regression tests', () => {
}
let
_tokenId
=
0
;
class
IndicisiveModeState
implements
IState
{
clone
():
IState
{
return
this
;
}
equals
(
other
:
IState
):
boolean
{
return
true
;
}
getModeId
():
string
{
throw
new
Error
(
'
Not implemented
'
);
}
getStateData
():
IState
{
throw
new
Error
(
'
Not implemented
'
);
}
setStateData
(
state
:
IState
):
void
{
throw
new
Error
(
'
Not implemented
'
);
}
}
class
IndicisiveMode
extends
MockMode
{
constructor
()
{
super
();
TokenizationRegistry
.
register
(
this
.
getId
(),
{
getInitialState
:
()
=>
{
return
n
ull
;
return
n
ew
IndicisiveModeState
()
;
},
tokenize
:
(
line
,
state
,
offsetDelta
,
stopAtOffset
)
=>
{
let
myId
=
++
_tokenId
;
return
{
tokens
:
[
new
Token
(
0
,
'
custom.
'
+
myId
)],
actualStopOffset
:
line
.
length
,
endState
:
null
,
modeTransitions
:
[],
retokenize
:
null
endState
:
state
,
modeTransitions
:
[]
};
}
});
...
...
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
浏览文件 @
921fde65
...
...
@@ -5,12 +5,10 @@
'
use strict
'
;
import
*
as
assert
from
'
assert
'
;
import
{
TokenizationRegistry
}
from
'
vs/editor/common/modes
'
;
import
{
AbstractState
,
ITokenizationResult
}
from
'
vs/editor/common/modes/abstractState
'
;
import
{
TokenizationSupport
}
from
'
vs/editor/common/modes/supports/tokenizationSupport
'
;
import
{
TokenizationRegistry
,
IState
,
ILineTokens
}
from
'
vs/editor/common/modes
'
;
import
{
tokenizeToHtmlContent
}
from
'
vs/editor/common/modes/textToHtmlTokenizer
'
;
import
{
MockMode
}
from
'
vs/editor/test/common/mocks/mockMode
'
;
import
{
LineStream
}
from
'
vs/editor/common/modes/lineStream
'
;
import
{
Token
}
from
'
vs/editor/common/core/token
'
;
suite
(
'
Editor Modes - textToHtmlTokenizer
'
,
()
=>
{
test
(
'
TextToHtmlTokenizer
'
,
()
=>
{
...
...
@@ -58,28 +56,28 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
});
class
State
extends
AbstractState
{
constructor
(
modeId
:
string
)
{
super
(
modeId
);
}
public
makeClone
():
AbstractState
{
return
new
State
(
this
.
getModeId
());
}
public
tokenize
(
stream
:
LineStream
):
ITokenizationResult
{
let
chr
=
stream
.
peek
();
stream
.
advance
(
1
);
return
{
type
:
chr
===
'
.
'
?
''
:
'
text
'
};
}
}
class
Mode
extends
MockMode
{
constructor
()
{
super
();
TokenizationRegistry
.
register
(
this
.
getId
(),
new
TokenizationSupport
(
null
,
this
.
getId
(),
{
getInitialState
:
()
=>
new
State
(
this
.
getId
())
},
false
));
TokenizationRegistry
.
register
(
this
.
getId
(),
{
getInitialState
:
():
IState
=>
null
,
tokenize
:
(
line
:
string
,
state
:
IState
):
ILineTokens
=>
{
let
tokens
:
Token
[]
=
[];
for
(
let
i
=
0
;
i
<
line
.
length
;
i
++
)
{
let
chr
=
line
.
charAt
(
i
);
let
type
=
chr
===
'
.
'
?
''
:
'
text
'
;
if
(
tokens
.
length
>
0
&&
tokens
[
tokens
.
length
-
1
].
type
===
type
)
{
continue
;
}
tokens
.
push
(
new
Token
(
i
,
type
));
}
return
{
tokens
:
tokens
,
actualStopOffset
:
-
1
,
endState
:
null
,
modeTransitions
:
null
};
}
});
}
}
src/vs/monaco.d.ts
浏览文件 @
921fde65
...
...
@@ -4259,10 +4259,6 @@ declare module monaco.languages {
* A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned.
*/
endState
:
IState
;
/**
* An optional promise to force the model to retokenize this line (e.g. missing information at the point of tokenization)
*/
retokenize
?:
Promise
<
void
>
;
}
/**
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录