Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
int
Rust
提交
8cd0c088
R
Rust
项目概览
int
/
Rust
大约 1 年 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
R
Rust
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
8cd0c088
编写于
2月 20, 2017
作者:
J
Jeffrey Seyfried
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Introduce `syntax::parse::parser::TokenCursor`.
上级
8dca72be
变更
2
显示空白变更内容
内联
并排
Showing
2 changed file
with
173 addition
and
115 deletion
+173
-115
src/libsyntax/parse/parser.rs
src/libsyntax/parse/parser.rs
+127
-47
src/libsyntax/tokenstream.rs
src/libsyntax/tokenstream.rs
+46
-68
未找到文件。
src/libsyntax/parse/parser.rs
浏览文件 @
8cd0c088
...
...
@@ -9,7 +9,7 @@
// except according to those terms.
use
abi
::{
self
,
Abi
};
use
ast
::
BareFnTy
;
use
ast
::
{
AttrStyle
,
BareFnTy
}
;
use
ast
::{
RegionTyParamBound
,
TraitTyParamBound
,
TraitBoundModifier
};
use
ast
::
Unsafety
;
use
ast
::{
Mod
,
Arg
,
Arm
,
Attribute
,
BindingMode
,
TraitItemKind
};
...
...
@@ -46,21 +46,21 @@
use
parse
::{
self
,
classify
,
token
};
use
parse
::
common
::
SeqSep
;
use
parse
::
lexer
::
TokenAndSpan
;
use
parse
::
lexer
::
comments
::{
doc_comment_style
,
strip_doc_comment_decoration
};
use
parse
::
obsolete
::
ObsoleteSyntax
;
use
parse
::{
new_sub_parser_from_file
,
ParseSess
,
Directory
,
DirectoryOwnership
};
use
util
::
parser
::{
AssocOp
,
Fixity
};
use
print
::
pprust
;
use
ptr
::
P
;
use
parse
::
PResult
;
use
tokenstream
::{
Delimited
,
TokenTree
};
use
tokenstream
::{
self
,
Delimited
,
TokenTree
,
TokenStream
};
use
symbol
::{
Symbol
,
keywords
};
use
util
::
ThinVec
;
use
std
::
collections
::
HashSet
;
use
std
::
mem
;
use
std
::
{
cmp
,
mem
,
slice
}
;
use
std
::
path
::{
Path
,
PathBuf
};
use
std
::
rc
::
Rc
;
use
std
::
slice
;
bitflags!
{
flags
Restrictions
:
u8
{
...
...
@@ -175,12 +175,108 @@ pub struct Parser<'a> {
/// into modules, and sub-parsers have new values for this name.
pub
root_module_name
:
Option
<
String
>
,
pub
expected_tokens
:
Vec
<
TokenType
>
,
pub
tts
:
Vec
<
(
TokenTree
,
usize
)
>
,
token_cursor
:
TokenCursor
,
pub
desugar_doc_comments
:
bool
,
/// Whether we should configure out of line modules as we parse.
pub
cfg_mods
:
bool
,
}
struct
TokenCursor
{
frame
:
TokenCursorFrame
,
stack
:
Vec
<
TokenCursorFrame
>
,
}
struct
TokenCursorFrame
{
delim
:
token
::
DelimToken
,
span
:
Span
,
open_delim
:
bool
,
tree_cursor
:
tokenstream
::
Cursor
,
close_delim
:
bool
,
}
impl
TokenCursorFrame
{
fn
new
(
sp
:
Span
,
delimited
:
&
Delimited
)
->
Self
{
TokenCursorFrame
{
delim
:
delimited
.delim
,
span
:
sp
,
open_delim
:
delimited
.delim
==
token
::
NoDelim
,
tree_cursor
:
delimited
.tts
.iter
()
.cloned
()
.collect
::
<
TokenStream
>
()
.into_trees
(),
close_delim
:
delimited
.delim
==
token
::
NoDelim
,
}
}
}
impl
TokenCursor
{
fn
next
(
&
mut
self
)
->
TokenAndSpan
{
loop
{
let
tree
=
if
!
self
.frame.open_delim
{
self
.frame.open_delim
=
true
;
Delimited
{
delim
:
self
.frame.delim
,
tts
:
Vec
::
new
()
}
.open_tt
(
self
.frame.span
)
}
else
if
let
Some
(
tree
)
=
self
.frame.tree_cursor
.next
()
{
tree
}
else
if
!
self
.frame.close_delim
{
self
.frame.close_delim
=
true
;
Delimited
{
delim
:
self
.frame.delim
,
tts
:
Vec
::
new
()
}
.close_tt
(
self
.frame.span
)
}
else
if
let
Some
(
frame
)
=
self
.stack
.pop
()
{
self
.frame
=
frame
;
continue
}
else
{
return
TokenAndSpan
{
tok
:
token
::
Eof
,
sp
:
self
.frame.span
}
};
match
tree
{
TokenTree
::
Token
(
sp
,
tok
)
=>
return
TokenAndSpan
{
tok
:
tok
,
sp
:
sp
},
TokenTree
::
Delimited
(
sp
,
ref
delimited
)
=>
{
let
frame
=
TokenCursorFrame
::
new
(
sp
,
delimited
);
self
.stack
.push
(
mem
::
replace
(
&
mut
self
.frame
,
frame
));
}
}
}
}
fn
next_desugared
(
&
mut
self
)
->
TokenAndSpan
{
let
(
sp
,
name
)
=
match
self
.next
()
{
TokenAndSpan
{
sp
,
tok
:
token
::
DocComment
(
name
)
}
=>
(
sp
,
name
),
tok
@
_
=>
return
tok
,
};
let
stripped
=
strip_doc_comment_decoration
(
&
name
.as_str
());
// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
// required to wrap the text.
let
mut
num_of_hashes
=
0
;
let
mut
count
=
0
;
for
ch
in
stripped
.chars
()
{
count
=
match
ch
{
'"'
=>
1
,
'#'
if
count
>
0
=>
count
+
1
,
_
=>
0
,
};
num_of_hashes
=
cmp
::
max
(
num_of_hashes
,
count
);
}
let
body
=
TokenTree
::
Delimited
(
sp
,
Rc
::
new
(
Delimited
{
delim
:
token
::
Bracket
,
tts
:
vec!
[
TokenTree
::
Token
(
sp
,
token
::
Ident
(
ast
::
Ident
::
from_str
(
"doc"
))),
TokenTree
::
Token
(
sp
,
token
::
Eq
),
TokenTree
::
Token
(
sp
,
token
::
Literal
(
token
::
StrRaw
(
Symbol
::
intern
(
&
stripped
),
num_of_hashes
),
None
))],
}));
self
.stack
.push
(
mem
::
replace
(
&
mut
self
.frame
,
TokenCursorFrame
::
new
(
sp
,
&
Delimited
{
delim
:
token
::
NoDelim
,
tts
:
if
doc_comment_style
(
&
name
.as_str
())
==
AttrStyle
::
Inner
{
[
TokenTree
::
Token
(
sp
,
token
::
Pound
),
TokenTree
::
Token
(
sp
,
token
::
Not
),
body
]
.iter
()
.cloned
()
.collect
()
}
else
{
[
TokenTree
::
Token
(
sp
,
token
::
Pound
),
body
]
.iter
()
.cloned
()
.collect
()
},
})));
self
.next
()
}
}
#[derive(PartialEq,
Eq,
Clone)]
pub
enum
TokenType
{
Token
(
token
::
Token
),
...
...
@@ -313,10 +409,6 @@ pub fn new(sess: &'a ParseSess,
directory
:
Option
<
Directory
>
,
desugar_doc_comments
:
bool
)
->
Self
{
let
tt
=
TokenTree
::
Delimited
(
syntax_pos
::
DUMMY_SP
,
Rc
::
new
(
Delimited
{
delim
:
token
::
NoDelim
,
tts
:
tokens
,
}));
let
mut
parser
=
Parser
{
sess
:
sess
,
token
:
token
::
Underscore
,
...
...
@@ -328,7 +420,13 @@ pub fn new(sess: &'a ParseSess,
directory
:
Directory
{
path
:
PathBuf
::
new
(),
ownership
:
DirectoryOwnership
::
Owned
},
root_module_name
:
None
,
expected_tokens
:
Vec
::
new
(),
tts
:
if
tt
.len
()
>
0
{
vec!
[(
tt
,
0
)]
}
else
{
Vec
::
new
()
},
token_cursor
:
TokenCursor
{
frame
:
TokenCursorFrame
::
new
(
syntax_pos
::
DUMMY_SP
,
&
Delimited
{
delim
:
token
::
NoDelim
,
tts
:
tokens
,
}),
stack
:
Vec
::
new
(),
},
desugar_doc_comments
:
desugar_doc_comments
,
cfg_mods
:
true
,
};
...
...
@@ -346,28 +444,9 @@ pub fn new(sess: &'a ParseSess,
}
fn
next_tok
(
&
mut
self
)
->
TokenAndSpan
{
loop
{
let
tok
=
if
let
Some
((
tts
,
i
))
=
self
.tts
.pop
()
{
let
tt
=
tts
.get_tt
(
i
);
if
i
+
1
<
tts
.len
()
{
self
.tts
.push
((
tts
,
i
+
1
));
}
if
let
TokenTree
::
Token
(
sp
,
tok
)
=
tt
{
TokenAndSpan
{
tok
:
tok
,
sp
:
sp
}
}
else
{
self
.tts
.push
((
tt
,
0
));
continue
}
}
else
{
TokenAndSpan
{
tok
:
token
::
Eof
,
sp
:
self
.span
}
};
match
tok
.tok
{
token
::
DocComment
(
name
)
if
self
.desugar_doc_comments
=>
{
self
.tts
.push
((
TokenTree
::
Token
(
tok
.sp
,
token
::
DocComment
(
name
)),
0
));
}
_
=>
return
tok
,
}
match
self
.desugar_doc_comments
{
true
=>
self
.token_cursor
.next_desugared
(),
false
=>
self
.token_cursor
.next
(),
}
}
...
...
@@ -972,19 +1051,16 @@ pub fn look_ahead<R, F>(&mut self, dist: usize, f: F) -> R where
F
:
FnOnce
(
&
token
::
Token
)
->
R
,
{
if
dist
==
0
{
return
f
(
&
self
.token
)
;
return
f
(
&
self
.token
)
}
let
mut
tok
=
token
::
Eof
;
if
let
Some
(
&
(
ref
tts
,
mut
i
))
=
self
.tts
.last
()
{
i
+=
dist
-
1
;
if
i
<
tts
.len
()
{
tok
=
match
tts
.get_tt
(
i
)
{
f
(
&
match
self
.token_cursor.frame.tree_cursor
.look_ahead
(
dist
-
1
)
{
Some
(
tree
)
=>
match
tree
{
TokenTree
::
Token
(
_
,
tok
)
=>
tok
,
TokenTree
::
Delimited
(
_
,
delimited
)
=>
token
::
OpenDelim
(
delimited
.delim
),
};
}
}
f
(
&
tok
)
},
None
=>
token
::
CloseDelim
(
self
.token_cursor.frame.delim
),
})
}
pub
fn
fatal
(
&
self
,
m
:
&
str
)
->
DiagnosticBuilder
<
'a
>
{
self
.sess.span_diagnostic
.struct_span_fatal
(
self
.span
,
m
)
...
...
@@ -2569,10 +2645,14 @@ pub fn check_unknown_macro_variable(&mut self) {
pub
fn
parse_token_tree
(
&
mut
self
)
->
PResult
<
'a
,
TokenTree
>
{
match
self
.token
{
token
::
OpenDelim
(
..
)
=>
{
let
tt
=
self
.tts
.pop
()
.unwrap
()
.0
;
self
.span
=
tt
.span
();
let
frame
=
mem
::
replace
(
&
mut
self
.token_cursor.frame
,
self
.token_cursor.stack
.pop
()
.unwrap
());
self
.span
=
frame
.span
;
self
.bump
();
return
Ok
(
tt
);
return
Ok
(
TokenTree
::
Delimited
(
frame
.span
,
Rc
::
new
(
Delimited
{
delim
:
frame
.delim
,
tts
:
frame
.tree_cursor
.original_stream
()
.trees
()
.collect
(),
})));
},
token
::
CloseDelim
(
_
)
|
token
::
Eof
=>
unreachable!
(),
_
=>
Ok
(
TokenTree
::
Token
(
self
.span
,
self
.bump_and_get
())),
...
...
src/libsyntax/tokenstream.rs
浏览文件 @
8cd0c088
...
...
@@ -22,12 +22,11 @@
//! and a borrowed TokenStream is sufficient to build an owned TokenStream without taking
//! ownership of the original.
use
ast
::{
self
,
AttrStyle
,
LitKind
};
use
ast
::{
self
,
LitKind
};
use
syntax_pos
::{
BytePos
,
Span
,
DUMMY_SP
};
use
codemap
::
Spanned
;
use
ext
::
base
;
use
ext
::
tt
::{
macro_parser
,
quoted
};
use
parse
::
lexer
::
comments
::{
doc_comment_style
,
strip_doc_comment_decoration
};
use
parse
::{
self
,
Directory
};
use
parse
::
token
::{
self
,
Token
,
Lit
};
use
print
::
pprust
;
...
...
@@ -103,72 +102,6 @@ pub enum TokenTree {
}
impl
TokenTree
{
pub
fn
len
(
&
self
)
->
usize
{
match
*
self
{
TokenTree
::
Token
(
_
,
token
::
DocComment
(
name
))
=>
{
match
doc_comment_style
(
&
name
.as_str
())
{
AttrStyle
::
Outer
=>
2
,
AttrStyle
::
Inner
=>
3
,
}
}
TokenTree
::
Delimited
(
_
,
ref
delimed
)
=>
match
delimed
.delim
{
token
::
NoDelim
=>
delimed
.tts
.len
(),
_
=>
delimed
.tts
.len
()
+
2
,
},
TokenTree
::
Token
(
..
)
=>
0
,
}
}
pub
fn
get_tt
(
&
self
,
index
:
usize
)
->
TokenTree
{
match
(
self
,
index
)
{
(
&
TokenTree
::
Token
(
sp
,
token
::
DocComment
(
_
)),
0
)
=>
TokenTree
::
Token
(
sp
,
token
::
Pound
),
(
&
TokenTree
::
Token
(
sp
,
token
::
DocComment
(
name
)),
1
)
if
doc_comment_style
(
&
name
.as_str
())
==
AttrStyle
::
Inner
=>
{
TokenTree
::
Token
(
sp
,
token
::
Not
)
}
(
&
TokenTree
::
Token
(
sp
,
token
::
DocComment
(
name
)),
_
)
=>
{
let
stripped
=
strip_doc_comment_decoration
(
&
name
.as_str
());
// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
// required to wrap the text.
let
num_of_hashes
=
stripped
.chars
()
.scan
(
0
,
|
cnt
,
x
|
{
*
cnt
=
if
x
==
'"'
{
1
}
else
if
*
cnt
!=
0
&&
x
==
'#'
{
*
cnt
+
1
}
else
{
0
};
Some
(
*
cnt
)
})
.max
()
.unwrap_or
(
0
);
TokenTree
::
Delimited
(
sp
,
Rc
::
new
(
Delimited
{
delim
:
token
::
Bracket
,
tts
:
vec!
[
TokenTree
::
Token
(
sp
,
token
::
Ident
(
ast
::
Ident
::
from_str
(
"doc"
))),
TokenTree
::
Token
(
sp
,
token
::
Eq
),
TokenTree
::
Token
(
sp
,
token
::
Literal
(
token
::
StrRaw
(
Symbol
::
intern
(
&
stripped
),
num_of_hashes
),
None
))],
}))
}
(
&
TokenTree
::
Delimited
(
_
,
ref
delimed
),
_
)
if
delimed
.delim
==
token
::
NoDelim
=>
{
delimed
.tts
[
index
]
.clone
()
}
(
&
TokenTree
::
Delimited
(
span
,
ref
delimed
),
_
)
=>
{
if
index
==
0
{
return
delimed
.open_tt
(
span
);
}
if
index
==
delimed
.tts
.len
()
+
1
{
return
delimed
.close_tt
(
span
);
}
delimed
.tts
[
index
-
1
]
.clone
()
}
_
=>
panic!
(
"Cannot expand a token tree"
),
}
}
/// Use this token tree as a matcher to parse given tts.
pub
fn
parse
(
cx
:
&
base
::
ExtCtxt
,
mtch
:
&
[
quoted
::
TokenTree
],
tts
:
&
[
TokenTree
])
->
macro_parser
::
NamedParseResult
{
...
...
@@ -416,6 +349,51 @@ fn new(stream: TokenStream) -> Self {
}
})
}
pub
fn
original_stream
(
self
)
->
TokenStream
{
match
self
.0
{
CursorKind
::
Empty
=>
TokenStream
::
empty
(),
CursorKind
::
Tree
(
tree
,
_
)
=>
tree
.into
(),
CursorKind
::
Stream
(
cursor
)
=>
TokenStream
::
concat_rc_slice
({
cursor
.stack
.get
(
0
)
.cloned
()
.map
(|(
stream
,
_
)|
stream
)
.unwrap_or
(
cursor
.stream
)
}),
}
}
pub
fn
look_ahead
(
&
self
,
n
:
usize
)
->
Option
<
TokenTree
>
{
fn
look_ahead
(
streams
:
&
[
TokenStream
],
mut
n
:
usize
)
->
Result
<
TokenTree
,
usize
>
{
for
stream
in
streams
{
n
=
match
stream
.kind
{
TokenStreamKind
::
Tree
(
ref
tree
)
if
n
==
0
=>
return
Ok
(
tree
.clone
()),
TokenStreamKind
::
Tree
(
..
)
=>
n
-
1
,
TokenStreamKind
::
Stream
(
ref
stream
)
=>
match
look_ahead
(
stream
,
n
)
{
Ok
(
tree
)
=>
return
Ok
(
tree
),
Err
(
n
)
=>
n
,
},
_
=>
n
,
};
}
Err
(
n
)
}
match
self
.0
{
CursorKind
::
Empty
|
CursorKind
::
Tree
(
_
,
true
)
=>
Err
(
n
),
CursorKind
::
Tree
(
ref
tree
,
false
)
=>
look_ahead
(
&
[
tree
.clone
()
.into
()],
n
),
CursorKind
::
Stream
(
ref
cursor
)
=>
{
look_ahead
(
&
cursor
.stream
[
cursor
.index
..
],
n
)
.or_else
(|
mut
n
|
{
for
&
(
ref
stream
,
index
)
in
cursor
.stack
.iter
()
.rev
()
{
n
=
match
look_ahead
(
&
stream
[
index
..
],
n
)
{
Ok
(
tree
)
=>
return
Ok
(
tree
),
Err
(
n
)
=>
n
,
}
}
Err
(
n
)
})
}
}
.ok
()
}
}
impl
fmt
::
Display
for
TokenStream
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录