Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
xxadev
vscode
提交
5cc0aa28
V
vscode
项目概览
xxadev
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
5cc0aa28
编写于
12月 30, 2020
作者:
A
Alexandru Dima
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add support for including line feeds at the end of lines (microsoft/monaco-editor#2265)
上级
4a121608
变更
23
显示空白变更内容
内联
并排
Showing
23 changed file
with
130 addition
and
59 deletion
+130
-59
src/vs/editor/common/model/textModelTokens.ts
src/vs/editor/common/model/textModelTokens.ts
+5
-5
src/vs/editor/common/modes.ts
src/vs/editor/common/modes.ts
+2
-2
src/vs/editor/common/modes/textToHtmlTokenizer.ts
src/vs/editor/common/modes/textToHtmlTokenizer.ts
+3
-3
src/vs/editor/contrib/comment/test/lineCommentCommand.test.ts
...vs/editor/contrib/comment/test/lineCommentCommand.test.ts
+1
-1
src/vs/editor/contrib/suggest/test/suggestModel.test.ts
src/vs/editor/contrib/suggest/test/suggestModel.test.ts
+1
-1
src/vs/editor/standalone/browser/colorizer.ts
src/vs/editor/standalone/browser/colorizer.ts
+1
-1
src/vs/editor/standalone/browser/inspectTokens/inspectTokens.ts
.../editor/standalone/browser/inspectTokens/inspectTokens.ts
+5
-5
src/vs/editor/standalone/browser/standaloneEditor.ts
src/vs/editor/standalone/browser/standaloneEditor.ts
+2
-2
src/vs/editor/standalone/browser/standaloneLanguages.ts
src/vs/editor/standalone/browser/standaloneLanguages.ts
+4
-4
src/vs/editor/standalone/common/monarch/monarchCommon.ts
src/vs/editor/standalone/common/monarch/monarchCommon.ts
+1
-0
src/vs/editor/standalone/common/monarch/monarchCompile.ts
src/vs/editor/standalone/common/monarch/monarchCompile.ts
+2
-0
src/vs/editor/standalone/common/monarch/monarchLexer.ts
src/vs/editor/standalone/common/monarch/monarchLexer.ts
+24
-20
src/vs/editor/standalone/common/monarch/monarchTypes.ts
src/vs/editor/standalone/common/monarch/monarchTypes.ts
+5
-0
src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts
...ditor/standalone/test/browser/standaloneLanguages.test.ts
+2
-2
src/vs/editor/standalone/test/monarch/monarch.test.ts
src/vs/editor/standalone/test/monarch/monarch.test.ts
+56
-2
src/vs/editor/test/browser/controller/cursor.test.ts
src/vs/editor/test/browser/controller/cursor.test.ts
+1
-1
src/vs/editor/test/common/model/model.modes.test.ts
src/vs/editor/test/common/model/model.modes.test.ts
+2
-2
src/vs/editor/test/common/model/model.test.ts
src/vs/editor/test/common/model/model.test.ts
+1
-1
src/vs/editor/test/common/model/textModelWithTokens.test.ts
src/vs/editor/test/common/model/textModelWithTokens.test.ts
+3
-3
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
+1
-1
src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts
...editor/test/common/viewModel/splitLinesCollection.test.ts
+1
-1
src/vs/monaco.d.ts
src/vs/monaco.d.ts
+5
-0
src/vs/workbench/services/textMate/browser/abstractTextMateService.ts
...ench/services/textMate/browser/abstractTextMateService.ts
+2
-2
未找到文件。
src/vs/editor/common/model/textModelTokens.ts
浏览文件 @
5cc0aa28
...
...
@@ -359,7 +359,7 @@ export class TextModelTokenization extends Disposable {
const
text
=
this
.
_textModel
.
getLineContent
(
lineIndex
+
1
);
const
lineStartState
=
this
.
_tokenizationStateStore
.
getBeginState
(
lineIndex
);
const
r
=
safeTokenize
(
languageIdentifier
,
this
.
_tokenizationSupport
,
text
,
lineStartState
!
);
const
r
=
safeTokenize
(
languageIdentifier
,
this
.
_tokenizationSupport
,
text
,
true
,
lineStartState
!
);
builder
.
add
(
lineIndex
+
1
,
r
.
tokens
);
this
.
_tokenizationStateStore
.
setEndState
(
linesLength
,
lineIndex
,
r
.
endState
);
lineIndex
=
this
.
_tokenizationStateStore
.
invalidLineStartIndex
-
1
;
// -1 because the outer loop increments it
...
...
@@ -410,13 +410,13 @@ export class TextModelTokenization extends Disposable {
const
languageIdentifier
=
this
.
_textModel
.
getLanguageIdentifier
();
let
state
=
initialState
;
for
(
let
i
=
fakeLines
.
length
-
1
;
i
>=
0
;
i
--
)
{
let
r
=
safeTokenize
(
languageIdentifier
,
this
.
_tokenizationSupport
,
fakeLines
[
i
],
state
);
let
r
=
safeTokenize
(
languageIdentifier
,
this
.
_tokenizationSupport
,
fakeLines
[
i
],
false
,
state
);
state
=
r
.
endState
;
}
for
(
let
lineNumber
=
startLineNumber
;
lineNumber
<=
endLineNumber
;
lineNumber
++
)
{
let
text
=
this
.
_textModel
.
getLineContent
(
lineNumber
);
let
r
=
safeTokenize
(
languageIdentifier
,
this
.
_tokenizationSupport
,
text
,
state
);
let
r
=
safeTokenize
(
languageIdentifier
,
this
.
_tokenizationSupport
,
text
,
true
,
state
);
builder
.
add
(
lineNumber
,
r
.
tokens
);
this
.
_tokenizationStateStore
.
setFakeTokens
(
lineNumber
-
1
);
state
=
r
.
endState
;
...
...
@@ -443,12 +443,12 @@ function initializeTokenization(textModel: TextModel): [ITokenizationSupport | n
return
[
tokenizationSupport
,
initialState
];
}
function
safeTokenize
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
|
null
,
text
:
string
,
state
:
IState
):
TokenizationResult2
{
function
safeTokenize
(
languageIdentifier
:
LanguageIdentifier
,
tokenizationSupport
:
ITokenizationSupport
|
null
,
text
:
string
,
hasEOL
:
boolean
,
state
:
IState
):
TokenizationResult2
{
let
r
:
TokenizationResult2
|
null
=
null
;
if
(
tokenizationSupport
)
{
try
{
r
=
tokenizationSupport
.
tokenize2
(
text
,
state
.
clone
(),
0
);
r
=
tokenizationSupport
.
tokenize2
(
text
,
hasEOL
,
state
.
clone
(),
0
);
}
catch
(
e
)
{
onUnexpectedError
(
e
);
}
...
...
src/vs/editor/common/modes.ts
浏览文件 @
5cc0aa28
...
...
@@ -211,9 +211,9 @@ export interface ITokenizationSupport {
getInitialState
():
IState
;
// add offsetDelta to each of the returned indices
tokenize
(
line
:
string
,
state
:
IState
,
offsetDelta
:
number
):
TokenizationResult
;
tokenize
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
,
offsetDelta
:
number
):
TokenizationResult
;
tokenize2
(
line
:
string
,
state
:
IState
,
offsetDelta
:
number
):
TokenizationResult2
;
tokenize2
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
,
offsetDelta
:
number
):
TokenizationResult2
;
}
/**
...
...
src/vs/editor/common/modes/textToHtmlTokenizer.ts
浏览文件 @
5cc0aa28
...
...
@@ -12,12 +12,12 @@ import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
export
interface
IReducedTokenizationSupport
{
getInitialState
():
IState
;
tokenize2
(
line
:
string
,
state
:
IState
,
offsetDelta
:
number
):
TokenizationResult2
;
tokenize2
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
,
offsetDelta
:
number
):
TokenizationResult2
;
}
const
fallback
:
IReducedTokenizationSupport
=
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize2
:
(
buffer
:
string
,
state
:
IState
,
deltaOffset
:
number
)
=>
nullTokenize2
(
LanguageId
.
Null
,
buffer
,
state
,
deltaOffset
)
tokenize2
:
(
buffer
:
string
,
hasEOL
:
boolean
,
state
:
IState
,
deltaOffset
:
number
)
=>
nullTokenize2
(
LanguageId
.
Null
,
buffer
,
state
,
deltaOffset
)
};
export
function
tokenizeToString
(
text
:
string
,
tokenizationSupport
:
IReducedTokenizationSupport
=
fallback
):
string
{
...
...
@@ -110,7 +110,7 @@ function _tokenizeToString(text: string, tokenizationSupport: IReducedTokenizati
result
+=
`<br/>`
;
}
let
tokenizationResult
=
tokenizationSupport
.
tokenize2
(
line
,
currentState
,
0
);
let
tokenizationResult
=
tokenizationSupport
.
tokenize2
(
line
,
true
,
currentState
,
0
);
LineTokens
.
convertToEndOffset
(
tokenizationResult
.
tokens
,
line
.
length
);
let
lineTokens
=
new
LineTokens
(
tokenizationResult
.
tokens
,
line
);
let
viewLineTokens
=
lineTokens
.
inflate
();
...
...
src/vs/editor/contrib/comment/test/lineCommentCommand.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -1083,7 +1083,7 @@ suite('Editor Contrib - Line Comment in mixed modes', () => {
tokenize
:
()
=>
{
throw
new
Error
(
'
not implemented
'
);
},
tokenize2
:
(
line
:
string
,
state
:
modes
.
IState
):
TokenizationResult2
=>
{
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
):
TokenizationResult2
=>
{
let
languageId
=
(
/^ /
.
test
(
line
)
?
INNER_LANGUAGE_ID
:
OUTER_LANGUAGE_ID
);
let
tokens
=
new
Uint32Array
(
1
<<
1
);
...
...
src/vs/editor/contrib/suggest/test/suggestModel.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -71,7 +71,7 @@ suite('SuggestModel - Context', function () {
this
.
_register
(
TokenizationRegistry
.
register
(
this
.
getLanguageIdentifier
().
language
,
{
getInitialState
:
():
IState
=>
NULL_STATE
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
:
string
,
state
:
IState
):
TokenizationResult2
=>
{
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
):
TokenizationResult2
=>
{
const
tokensArr
:
number
[]
=
[];
let
prevLanguageId
:
LanguageIdentifier
|
undefined
=
undefined
;
for
(
let
i
=
0
;
i
<
line
.
length
;
i
++
)
{
...
...
src/vs/editor/standalone/browser/colorizer.ts
浏览文件 @
5cc0aa28
...
...
@@ -222,7 +222,7 @@ function _actualColorize(lines: string[], tabSize: number, tokenizationSupport:
for
(
let
i
=
0
,
length
=
lines
.
length
;
i
<
length
;
i
++
)
{
let
line
=
lines
[
i
];
let
tokenizeResult
=
tokenizationSupport
.
tokenize2
(
line
,
state
,
0
);
let
tokenizeResult
=
tokenizationSupport
.
tokenize2
(
line
,
true
,
state
,
0
);
LineTokens
.
convertToEndOffset
(
tokenizeResult
.
tokens
,
line
.
length
);
let
lineTokens
=
new
LineTokens
(
tokenizeResult
.
tokens
,
line
);
const
isBasicASCII
=
ViewLineRenderingData
.
isBasicASCII
(
line
,
/* check for basic ASCII */
true
);
...
...
src/vs/editor/standalone/browser/inspectTokens/inspectTokens.ts
浏览文件 @
5cc0aa28
...
...
@@ -137,8 +137,8 @@ function getSafeTokenizationSupport(languageIdentifier: LanguageIdentifier): ITo
}
return
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize
:
(
line
:
string
,
state
:
IState
,
deltaOffset
:
number
)
=>
nullTokenize
(
languageIdentifier
.
language
,
line
,
state
,
deltaOffset
),
tokenize2
:
(
line
:
string
,
state
:
IState
,
deltaOffset
:
number
)
=>
nullTokenize2
(
languageIdentifier
.
id
,
line
,
state
,
deltaOffset
)
tokenize
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
,
deltaOffset
:
number
)
=>
nullTokenize
(
languageIdentifier
.
language
,
line
,
state
,
deltaOffset
),
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
,
deltaOffset
:
number
)
=>
nullTokenize2
(
languageIdentifier
.
id
,
line
,
state
,
deltaOffset
)
};
}
...
...
@@ -293,8 +293,8 @@ class InspectTokensWidget extends Disposable implements IContentWidget {
private
_getTokensAtLine
(
lineNumber
:
number
):
ICompleteLineTokenization
{
let
stateBeforeLine
=
this
.
_getStateBeforeLine
(
lineNumber
);
let
tokenizationResult1
=
this
.
_tokenizationSupport
.
tokenize
(
this
.
_model
.
getLineContent
(
lineNumber
),
stateBeforeLine
,
0
);
let
tokenizationResult2
=
this
.
_tokenizationSupport
.
tokenize2
(
this
.
_model
.
getLineContent
(
lineNumber
),
stateBeforeLine
,
0
);
let
tokenizationResult1
=
this
.
_tokenizationSupport
.
tokenize
(
this
.
_model
.
getLineContent
(
lineNumber
),
true
,
stateBeforeLine
,
0
);
let
tokenizationResult2
=
this
.
_tokenizationSupport
.
tokenize2
(
this
.
_model
.
getLineContent
(
lineNumber
),
true
,
stateBeforeLine
,
0
);
return
{
startState
:
stateBeforeLine
,
...
...
@@ -308,7 +308,7 @@ class InspectTokensWidget extends Disposable implements IContentWidget {
let
state
:
IState
=
this
.
_tokenizationSupport
.
getInitialState
();
for
(
let
i
=
1
;
i
<
lineNumber
;
i
++
)
{
let
tokenizationResult
=
this
.
_tokenizationSupport
.
tokenize
(
this
.
_model
.
getLineContent
(
i
),
state
,
0
);
let
tokenizationResult
=
this
.
_tokenizationSupport
.
tokenize
(
this
.
_model
.
getLineContent
(
i
),
true
,
state
,
0
);
state
=
tokenizationResult
.
endState
;
}
...
...
src/vs/editor/standalone/browser/standaloneEditor.ts
浏览文件 @
5cc0aa28
...
...
@@ -276,7 +276,7 @@ function getSafeTokenizationSupport(language: string): Omit<modes.ITokenizationS
}
return
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize
:
(
line
:
string
,
state
:
modes
.
IState
,
deltaOffset
:
number
)
=>
nullTokenize
(
language
,
line
,
state
,
deltaOffset
)
tokenize
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
,
deltaOffset
:
number
)
=>
nullTokenize
(
language
,
line
,
state
,
deltaOffset
)
};
}
...
...
@@ -294,7 +294,7 @@ export function tokenize(text: string, languageId: string): Token[][] {
let
state
=
tokenizationSupport
.
getInitialState
();
for
(
let
i
=
0
,
len
=
lines
.
length
;
i
<
len
;
i
++
)
{
let
line
=
lines
[
i
];
let
tokenizationResult
=
tokenizationSupport
.
tokenize
(
line
,
state
,
0
);
let
tokenizationResult
=
tokenizationSupport
.
tokenize
(
line
,
true
,
state
,
0
);
result
[
i
]
=
tokenizationResult
.
tokens
;
state
=
tokenizationResult
.
endState
;
...
...
src/vs/editor/standalone/browser/standaloneLanguages.ts
浏览文件 @
5cc0aa28
...
...
@@ -88,14 +88,14 @@ export class EncodedTokenizationSupport2Adapter implements modes.ITokenizationSu
return
this
.
_actual
.
getInitialState
();
}
public
tokenize
(
line
:
string
,
state
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult
{
public
tokenize
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult
{
if
(
typeof
this
.
_actual
.
tokenize
===
'
function
'
)
{
return
TokenizationSupport2Adapter
.
adaptTokenize
(
this
.
_languageIdentifier
.
language
,
<
{
tokenize
(
line
:
string
,
state
:
modes
.
IState
):
ILineTokens
;
}
>
this
.
_actual
,
line
,
state
,
offsetDelta
);
}
throw
new
Error
(
'
Not supported!
'
);
}
public
tokenize2
(
line
:
string
,
state
:
modes
.
IState
):
TokenizationResult2
{
public
tokenize2
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
):
TokenizationResult2
{
let
result
=
this
.
_actual
.
tokenizeEncoded
(
line
,
state
);
return
new
TokenizationResult2
(
result
.
tokens
,
result
.
endState
);
}
...
...
@@ -158,7 +158,7 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
return
new
TokenizationResult
(
tokens
,
endState
);
}
public
tokenize
(
line
:
string
,
state
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult
{
public
tokenize
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult
{
return
TokenizationSupport2Adapter
.
adaptTokenize
(
this
.
_languageIdentifier
.
language
,
this
.
_actual
,
line
,
state
,
offsetDelta
);
}
...
...
@@ -200,7 +200,7 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
return
actualResult
;
}
public
tokenize2
(
line
:
string
,
state
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult2
{
public
tokenize2
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult2
{
let
actualResult
=
this
.
_actual
.
tokenize
(
line
,
state
);
let
tokens
=
this
.
_toBinaryTokens
(
actualResult
.
tokens
,
offsetDelta
);
...
...
src/vs/editor/standalone/common/monarch/monarchCommon.ts
浏览文件 @
5cc0aa28
...
...
@@ -22,6 +22,7 @@ export const enum MonarchBracket {
export
interface
ILexerMin
{
languageId
:
string
;
includeLF
:
boolean
;
noThrow
:
boolean
;
ignoreCase
:
boolean
;
unicode
:
boolean
;
...
...
src/vs/editor/standalone/common/monarch/monarchCompile.ts
浏览文件 @
5cc0aa28
...
...
@@ -395,6 +395,7 @@ export function compile(languageId: string, json: IMonarchLanguage): monarchComm
// Create our lexer
let
lexer
:
monarchCommon
.
ILexer
=
<
monarchCommon
.
ILexer
>
{};
lexer
.
languageId
=
languageId
;
lexer
.
includeLF
=
bool
(
json
.
includeLF
,
false
);
lexer
.
noThrow
=
false
;
// raise exceptions during compilation
lexer
.
maxStack
=
100
;
...
...
@@ -411,6 +412,7 @@ export function compile(languageId: string, json: IMonarchLanguage): monarchComm
// For calling compileAction later on
let
lexerMin
:
monarchCommon
.
ILexerMin
=
<
any
>
json
;
lexerMin
.
languageId
=
languageId
;
lexerMin
.
includeLF
=
lexer
.
includeLF
;
lexerMin
.
ignoreCase
=
lexer
.
ignoreCase
;
lexerMin
.
unicode
=
lexer
.
unicode
;
lexerMin
.
noThrow
=
lexer
.
noThrow
;
...
...
src/vs/editor/standalone/common/monarch/monarchLexer.ts
浏览文件 @
5cc0aa28
...
...
@@ -231,7 +231,7 @@ class MonarchLineState implements modes.IState {
interface
IMonarchTokensCollector
{
enterMode
(
startOffset
:
number
,
modeId
:
string
):
void
;
emit
(
startOffset
:
number
,
type
:
string
):
void
;
nestedModeTokenize
(
embeddedModeLine
:
string
,
embeddedModeData
:
EmbeddedModeData
,
offsetDelta
:
number
):
modes
.
IState
;
nestedModeTokenize
(
embeddedModeLine
:
string
,
hasEOL
:
boolean
,
embeddedModeData
:
EmbeddedModeData
,
offsetDelta
:
number
):
modes
.
IState
;
}
class
MonarchClassicTokensCollector
implements
IMonarchTokensCollector
{
...
...
@@ -261,7 +261,7 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector {
this
.
_tokens
.
push
(
new
Token
(
startOffset
,
type
,
this
.
_language
!
));
}
public
nestedModeTokenize
(
embeddedModeLine
:
string
,
embeddedModeData
:
EmbeddedModeData
,
offsetDelta
:
number
):
modes
.
IState
{
public
nestedModeTokenize
(
embeddedModeLine
:
string
,
hasEOL
:
boolean
,
embeddedModeData
:
EmbeddedModeData
,
offsetDelta
:
number
):
modes
.
IState
{
const
nestedModeId
=
embeddedModeData
.
modeId
;
const
embeddedModeState
=
embeddedModeData
.
state
;
...
...
@@ -272,7 +272,7 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector {
return
embeddedModeState
;
}
let
nestedResult
=
nestedModeTokenizationSupport
.
tokenize
(
embeddedModeLine
,
embeddedModeState
,
offsetDelta
);
let
nestedResult
=
nestedModeTokenizationSupport
.
tokenize
(
embeddedModeLine
,
hasEOL
,
embeddedModeState
,
offsetDelta
);
this
.
_tokens
=
this
.
_tokens
.
concat
(
nestedResult
.
tokens
);
this
.
_lastTokenType
=
null
;
this
.
_lastTokenLanguage
=
null
;
...
...
@@ -345,7 +345,7 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector {
return
result
;
}
public
nestedModeTokenize
(
embeddedModeLine
:
string
,
embeddedModeData
:
EmbeddedModeData
,
offsetDelta
:
number
):
modes
.
IState
{
public
nestedModeTokenize
(
embeddedModeLine
:
string
,
hasEOL
:
boolean
,
embeddedModeData
:
EmbeddedModeData
,
offsetDelta
:
number
):
modes
.
IState
{
const
nestedModeId
=
embeddedModeData
.
modeId
;
const
embeddedModeState
=
embeddedModeData
.
state
;
...
...
@@ -356,7 +356,7 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector {
return
embeddedModeState
;
}
let
nestedResult
=
nestedModeTokenizationSupport
.
tokenize2
(
embeddedModeLine
,
embeddedModeState
,
offsetDelta
);
let
nestedResult
=
nestedModeTokenizationSupport
.
tokenize2
(
embeddedModeLine
,
hasEOL
,
embeddedModeState
,
offsetDelta
);
this
.
_prependTokens
=
MonarchModernTokensCollector
.
_merge
(
this
.
_prependTokens
,
this
.
_tokens
,
nestedResult
.
tokens
);
this
.
_tokens
=
[];
this
.
_currentLanguageId
=
0
;
...
...
@@ -456,23 +456,23 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
return
MonarchLineStateFactory
.
create
(
rootState
,
null
);
}
public
tokenize
(
line
:
string
,
lineState
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult
{
public
tokenize
(
line
:
string
,
hasEOL
:
boolean
,
lineState
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult
{
let
tokensCollector
=
new
MonarchClassicTokensCollector
();
let
endLineState
=
this
.
_tokenize
(
line
,
<
MonarchLineState
>
lineState
,
offsetDelta
,
tokensCollector
);
let
endLineState
=
this
.
_tokenize
(
line
,
hasEOL
,
<
MonarchLineState
>
lineState
,
offsetDelta
,
tokensCollector
);
return
tokensCollector
.
finalize
(
endLineState
);
}
public
tokenize2
(
line
:
string
,
lineState
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult2
{
public
tokenize2
(
line
:
string
,
hasEOL
:
boolean
,
lineState
:
modes
.
IState
,
offsetDelta
:
number
):
TokenizationResult2
{
let
tokensCollector
=
new
MonarchModernTokensCollector
(
this
.
_modeService
,
this
.
_standaloneThemeService
.
getColorTheme
().
tokenTheme
);
let
endLineState
=
this
.
_tokenize
(
line
,
<
MonarchLineState
>
lineState
,
offsetDelta
,
tokensCollector
);
let
endLineState
=
this
.
_tokenize
(
line
,
hasEOL
,
<
MonarchLineState
>
lineState
,
offsetDelta
,
tokensCollector
);
return
tokensCollector
.
finalize
(
endLineState
);
}
private
_tokenize
(
line
:
string
,
lineState
:
MonarchLineState
,
offsetDelta
:
number
,
collector
:
IMonarchTokensCollector
):
MonarchLineState
{
private
_tokenize
(
line
:
string
,
hasEOL
:
boolean
,
lineState
:
MonarchLineState
,
offsetDelta
:
number
,
collector
:
IMonarchTokensCollector
):
MonarchLineState
{
if
(
lineState
.
embeddedModeData
)
{
return
this
.
_nestedTokenize
(
line
,
lineState
,
offsetDelta
,
collector
);
return
this
.
_nestedTokenize
(
line
,
hasEOL
,
lineState
,
offsetDelta
,
collector
);
}
else
{
return
this
.
_myTokenize
(
line
,
lineState
,
offsetDelta
,
collector
);
return
this
.
_myTokenize
(
line
,
hasEOL
,
lineState
,
offsetDelta
,
collector
);
}
}
...
...
@@ -518,24 +518,24 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
return
popOffset
;
}
private
_nestedTokenize
(
line
:
string
,
lineState
:
MonarchLineState
,
offsetDelta
:
number
,
tokensCollector
:
IMonarchTokensCollector
):
MonarchLineState
{
private
_nestedTokenize
(
line
:
string
,
hasEOL
:
boolean
,
lineState
:
MonarchLineState
,
offsetDelta
:
number
,
tokensCollector
:
IMonarchTokensCollector
):
MonarchLineState
{
let
popOffset
=
this
.
_findLeavingNestedModeOffset
(
line
,
lineState
);
if
(
popOffset
===
-
1
)
{
// tokenization will not leave nested mode
let
nestedEndState
=
tokensCollector
.
nestedModeTokenize
(
line
,
lineState
.
embeddedModeData
!
,
offsetDelta
);
let
nestedEndState
=
tokensCollector
.
nestedModeTokenize
(
line
,
hasEOL
,
lineState
.
embeddedModeData
!
,
offsetDelta
);
return
MonarchLineStateFactory
.
create
(
lineState
.
stack
,
new
EmbeddedModeData
(
lineState
.
embeddedModeData
!
.
modeId
,
nestedEndState
));
}
let
nestedModeLine
=
line
.
substring
(
0
,
popOffset
);
if
(
nestedModeLine
.
length
>
0
)
{
// tokenize with the nested mode
tokensCollector
.
nestedModeTokenize
(
nestedModeLine
,
lineState
.
embeddedModeData
!
,
offsetDelta
);
tokensCollector
.
nestedModeTokenize
(
nestedModeLine
,
false
,
lineState
.
embeddedModeData
!
,
offsetDelta
);
}
let
restOfTheLine
=
line
.
substring
(
popOffset
);
return
this
.
_myTokenize
(
restOfTheLine
,
lineState
,
offsetDelta
+
popOffset
,
tokensCollector
);
return
this
.
_myTokenize
(
restOfTheLine
,
hasEOL
,
lineState
,
offsetDelta
+
popOffset
,
tokensCollector
);
}
private
_safeRuleName
(
rule
:
monarchCommon
.
IRule
|
null
):
string
{
...
...
@@ -545,9 +545,11 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
return
'
(unknown)
'
;
}
private
_myTokenize
(
line
:
string
,
lineState
:
MonarchLineState
,
offsetDelta
:
number
,
tokensCollector
:
IMonarchTokensCollector
):
MonarchLineState
{
private
_myTokenize
(
line
WithoutLF
:
string
,
hasEOL
:
boolean
,
lineState
:
MonarchLineState
,
offsetDelta
:
number
,
tokensCollector
:
IMonarchTokensCollector
):
MonarchLineState
{
tokensCollector
.
enterMode
(
offsetDelta
,
this
.
_modeId
);
const
lineWithoutLFLength
=
lineWithoutLF
.
length
;
const
line
=
(
hasEOL
&&
this
.
_lexer
.
includeLF
?
lineWithoutLF
+
'
\n
'
:
lineWithoutLF
);
const
lineLength
=
line
.
length
;
let
embeddedModeData
=
lineState
.
embeddedModeData
;
...
...
@@ -752,8 +754,8 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
if
(
pos
<
lineLength
)
{
// there is content from the embedded mode on this line
const
restOfLine
=
line
.
substr
(
pos
);
return
this
.
_nestedTokenize
(
restOfLine
,
MonarchLineStateFactory
.
create
(
stack
,
embeddedModeData
),
offsetDelta
+
pos
,
tokensCollector
);
const
restOfLine
=
line
WithoutLF
.
substr
(
pos
);
return
this
.
_nestedTokenize
(
restOfLine
,
hasEOL
,
MonarchLineStateFactory
.
create
(
stack
,
embeddedModeData
),
offsetDelta
+
pos
,
tokensCollector
);
}
else
{
return
MonarchLineStateFactory
.
create
(
stack
,
embeddedModeData
);
}
...
...
@@ -831,8 +833,10 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
tokenType
=
monarchCommon
.
sanitize
(
token
);
}
if
(
pos0
<
lineWithoutLFLength
)
{
tokensCollector
.
emit
(
pos0
+
offsetDelta
,
tokenType
);
}
}
if
(
enteringEmbeddedMode
!==
null
)
{
return
computeNewStateForEmbeddedMode
(
enteringEmbeddedMode
);
...
...
src/vs/editor/standalone/common/monarch/monarchTypes.ts
浏览文件 @
5cc0aa28
...
...
@@ -41,6 +41,11 @@ export interface IMonarchLanguage {
* attach this to every token class (by default '.' + name)
*/
tokenPostfix
?:
string
;
/**
* include line feeds (in the form of a \n character) at the end of lines
* Defaults to false
*/
includeLF
?:
boolean
;
}
/**
...
...
src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -108,10 +108,10 @@ suite('TokenizationSupport2Adapter', () => {
const
adapter
=
new
TokenizationSupport2Adapter
(
new
MockThemeService
(),
languageIdentifier
,
new
BadTokensProvider
());
const
actualClassicTokens
=
adapter
.
tokenize
(
'
whatever
'
,
MockState
.
INSTANCE
,
offsetDelta
);
const
actualClassicTokens
=
adapter
.
tokenize
(
'
whatever
'
,
true
,
MockState
.
INSTANCE
,
offsetDelta
);
assert
.
deepEqual
(
actualClassicTokens
.
tokens
,
expectedClassicTokens
);
const
actualModernTokens
=
adapter
.
tokenize2
(
'
whatever
'
,
MockState
.
INSTANCE
,
offsetDelta
);
const
actualModernTokens
=
adapter
.
tokenize2
(
'
whatever
'
,
true
,
MockState
.
INSTANCE
,
offsetDelta
);
const
modernTokens
:
number
[]
=
[];
for
(
let
i
=
0
;
i
<
actualModernTokens
.
tokens
.
length
;
i
++
)
{
modernTokens
[
i
]
=
actualModernTokens
.
tokens
[
i
];
...
...
src/vs/editor/standalone/test/monarch/monarch.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -68,7 +68,7 @@ suite('Monarch', () => {
const
actualTokens
:
Token
[][]
=
[];
let
state
=
tokenizer
.
getInitialState
();
for
(
const
line
of
lines
)
{
const
result
=
tokenizer
.
tokenize
(
line
,
state
,
0
);
const
result
=
tokenizer
.
tokenize
(
line
,
true
,
state
,
0
);
actualTokens
.
push
(
result
.
tokens
);
state
=
result
.
endState
;
}
...
...
@@ -143,7 +143,7 @@ suite('Monarch', () => {
const
actualTokens
:
Token
[][]
=
[];
let
state
=
tokenizer
.
getInitialState
();
for
(
const
line
of
lines
)
{
const
result
=
tokenizer
.
tokenize
(
line
,
state
,
0
);
const
result
=
tokenizer
.
tokenize
(
line
,
true
,
state
,
0
);
actualTokens
.
push
(
result
.
tokens
);
state
=
result
.
endState
;
}
...
...
@@ -164,4 +164,58 @@ suite('Monarch', () => {
]);
});
test
(
'
microsoft/monaco-editor#2265: Exit a state at end of line
'
,
()
=>
{
const
modeService
=
new
ModeServiceImpl
();
const
tokenizer
=
createMonarchTokenizer
(
modeService
,
'
test
'
,
{
includeLF
:
true
,
tokenizer
:
{
root
:
[
[
/^
\*
/
,
''
,
'
@inner
'
],
[
/
\:\*
/
,
''
,
'
@inner
'
],
[
/
[^
*:
]
+/
,
'
string
'
],
[
/
[
*:
]
/
,
'
string
'
]
],
inner
:
[
[
/
\n
/
,
''
,
'
@pop
'
],
[
/
\d
+/
,
'
number
'
],
[
/
[^\d]
+/
,
''
]
]
}
});
const
lines
=
[
`PRINT 10 * 20`
,
`*FX200, 3`
,
`PRINT 2*3:*FX200, 3`
];
const
actualTokens
:
Token
[][]
=
[];
let
state
=
tokenizer
.
getInitialState
();
for
(
const
line
of
lines
)
{
const
result
=
tokenizer
.
tokenize
(
line
,
true
,
state
,
0
);
actualTokens
.
push
(
result
.
tokens
);
state
=
result
.
endState
;
}
assert
.
deepStrictEqual
(
actualTokens
,
[
[
new
Token
(
0
,
'
string.test
'
,
'
test
'
),
],
[
new
Token
(
0
,
''
,
'
test
'
),
new
Token
(
3
,
'
number.test
'
,
'
test
'
),
new
Token
(
6
,
''
,
'
test
'
),
new
Token
(
8
,
'
number.test
'
,
'
test
'
),
],
[
new
Token
(
0
,
'
string.test
'
,
'
test
'
),
new
Token
(
9
,
''
,
'
test
'
),
new
Token
(
13
,
'
number.test
'
,
'
test
'
),
new
Token
(
16
,
''
,
'
test
'
),
new
Token
(
18
,
'
number.test
'
,
'
test
'
),
]
]);
});
});
src/vs/editor/test/browser/controller/cursor.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -2406,7 +2406,7 @@ suite('Editor Controller - Regression tests', () => {
const
tokenizationSupport
:
ITokenizationSupport
=
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
:
string
,
state
:
IState
):
TokenizationResult2
=>
{
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
):
TokenizationResult2
=>
{
return
new
TokenizationResult2
(
new
Uint32Array
(
0
),
state
);
}
};
...
...
src/vs/editor/test/common/model/model.modes.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -28,7 +28,7 @@ suite('Editor Model - Model Modes 1', () => {
const
tokenizationSupport
:
modes
.
ITokenizationSupport
=
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
:
string
,
state
:
modes
.
IState
):
TokenizationResult2
=>
{
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
):
TokenizationResult2
=>
{
calledFor
.
push
(
line
.
charAt
(
0
));
return
new
TokenizationResult2
(
new
Uint32Array
(
0
),
state
);
}
...
...
@@ -181,7 +181,7 @@ suite('Editor Model - Model Modes 2', () => {
const
tokenizationSupport
:
modes
.
ITokenizationSupport
=
{
getInitialState
:
()
=>
new
ModelState2
(
''
),
tokenize
:
undefined
!
,
tokenize2
:
(
line
:
string
,
state
:
modes
.
IState
):
TokenizationResult2
=>
{
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
):
TokenizationResult2
=>
{
calledFor
.
push
(
line
);
(
<
ModelState2
>
state
).
prevLineContent
=
line
;
return
new
TokenizationResult2
(
new
Uint32Array
(
0
),
state
);
...
...
src/vs/editor/test/common/model/model.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -389,7 +389,7 @@ suite('Editor Model - Words', () => {
this
.
_register
(
TokenizationRegistry
.
register
(
this
.
getLanguageIdentifier
().
language
,
{
getInitialState
:
():
IState
=>
NULL_STATE
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
:
string
,
state
:
IState
):
TokenizationResult2
=>
{
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
):
TokenizationResult2
=>
{
const
tokensArr
:
number
[]
=
[];
let
prevLanguageId
:
LanguageIdentifier
|
undefined
=
undefined
;
for
(
let
i
=
0
;
i
<
line
.
length
;
i
++
)
{
...
...
src/vs/editor/test/common/model/textModelWithTokens.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -351,7 +351,7 @@ suite('TextModelWithTokens', () => {
const
tokenizationSupport
:
ITokenizationSupport
=
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
,
state
)
=>
{
tokenize2
:
(
line
,
hasEOL
,
state
)
=>
{
switch
(
line
)
{
case
'
function hello() {
'
:
{
const
tokens
=
new
Uint32Array
([
...
...
@@ -446,7 +446,7 @@ suite('TextModelWithTokens regression tests', () => {
const
tokenizationSupport
:
ITokenizationSupport
=
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
,
state
)
=>
{
tokenize2
:
(
line
,
hasEOL
,
state
)
=>
{
let
myId
=
++
_tokenId
;
let
tokens
=
new
Uint32Array
(
2
);
tokens
[
0
]
=
0
;
...
...
@@ -550,7 +550,7 @@ suite('TextModelWithTokens regression tests', () => {
const
tokenizationSupport
:
ITokenizationSupport
=
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
,
state
)
=>
{
tokenize2
:
(
line
,
hasEOL
,
state
)
=>
{
let
tokens
=
new
Uint32Array
(
2
);
tokens
[
0
]
=
0
;
tokens
[
1
]
=
(
...
...
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -287,7 +287,7 @@ class Mode extends MockMode {
this
.
_register
(
TokenizationRegistry
.
register
(
this
.
getId
(),
{
getInitialState
:
():
IState
=>
null
!
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
:
string
,
state
:
IState
):
TokenizationResult2
=>
{
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
):
TokenizationResult2
=>
{
let
tokensArr
:
number
[]
=
[];
let
prevColor
:
ColorId
=
-
1
;
for
(
let
i
=
0
;
i
<
line
.
length
;
i
++
)
{
...
...
src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts
浏览文件 @
5cc0aa28
...
...
@@ -333,7 +333,7 @@ suite('SplitLinesCollection', () => {
const
tokenizationSupport
:
modes
.
ITokenizationSupport
=
{
getInitialState
:
()
=>
NULL_STATE
,
tokenize
:
undefined
!
,
tokenize2
:
(
line
:
string
,
state
:
modes
.
IState
):
TokenizationResult2
=>
{
tokenize2
:
(
line
:
string
,
hasEOL
:
boolean
,
state
:
modes
.
IState
):
TokenizationResult2
=>
{
let
tokens
=
_tokens
[
_lineIndex
++
];
let
result
=
new
Uint32Array
(
2
*
tokens
.
length
);
...
...
src/vs/monaco.d.ts
浏览文件 @
5cc0aa28
...
...
@@ -6449,6 +6449,11 @@ declare namespace monaco.languages {
* attach this to every token class (by default '.' + name)
*/
tokenPostfix
?:
string
;
/**
* include line feeds (in the form of a \n character) at the end of lines
* Defaults to false
*/
includeLF
?:
boolean
;
}
/**
...
...
src/vs/workbench/services/textMate/browser/abstractTextMateService.ts
浏览文件 @
5cc0aa28
...
...
@@ -445,11 +445,11 @@ class TMTokenizationSupport implements ITokenizationSupport {
return
this
.
_actual
.
getInitialState
();
}
tokenize
(
line
:
string
,
state
:
IState
,
offsetDelta
:
number
):
TokenizationResult
{
tokenize
(
line
:
string
,
hasEOL
:
boolean
,
state
:
IState
,
offsetDelta
:
number
):
TokenizationResult
{
throw
new
Error
(
'
Not supported!
'
);
}
tokenize2
(
line
:
string
,
state
:
StackElement
,
offsetDelta
:
number
):
TokenizationResult2
{
tokenize2
(
line
:
string
,
hasEOL
:
boolean
,
state
:
StackElement
,
offsetDelta
:
number
):
TokenizationResult2
{
if
(
offsetDelta
!==
0
)
{
throw
new
Error
(
'
Unexpected: offsetDelta should be 0.
'
);
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录