Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
int
Rust
提交
ca2a50fa
R
Rust
项目概览
int
/
Rust
接近 1 年 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
R
Rust
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ca2a50fa
编写于
5月 19, 2019
作者:
V
Vadim Petrochenkov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
syntax: Turn `token::Lit` into a struct
上级
558559e7
变更
12
隐藏空白更改
内联
并排
Showing
12 changed file
with
275 addition
and
286 deletion
+275
-286
src/librustc/hir/print.rs
src/librustc/hir/print.rs
+1
-2
src/librustc/ich/impls_syntax.rs
src/librustc/ich/impls_syntax.rs
+18
-16
src/librustdoc/html/highlight.rs
src/librustdoc/html/highlight.rs
+7
-7
src/libsyntax/ast.rs
src/libsyntax/ast.rs
+0
-2
src/libsyntax/diagnostics/plugin.rs
src/libsyntax/diagnostics/plugin.rs
+2
-2
src/libsyntax/parse/lexer/mod.rs
src/libsyntax/parse/lexer/mod.rs
+51
-54
src/libsyntax/parse/literal.rs
src/libsyntax/parse/literal.rs
+71
-59
src/libsyntax/parse/parser.rs
src/libsyntax/parse/parser.rs
+23
-18
src/libsyntax/parse/token.rs
src/libsyntax/parse/token.rs
+51
-44
src/libsyntax/print/pprust.rs
src/libsyntax/print/pprust.rs
+18
-18
src/libsyntax_ext/assert.rs
src/libsyntax_ext/assert.rs
+6
-9
src/libsyntax_ext/proc_macro_server.rs
src/libsyntax_ext/proc_macro_server.rs
+27
-55
未找到文件。
src/librustc/hir/print.rs
浏览文件 @
ca2a50fa
...
...
@@ -1249,8 +1249,7 @@ fn print_expr_addr_of(&mut self,
fn
print_literal
(
&
mut
self
,
lit
:
&
hir
::
Lit
)
->
io
::
Result
<
()
>
{
self
.maybe_print_comment
(
lit
.span
.lo
())
?
;
let
(
token
,
suffix
)
=
lit
.node
.to_lit_token
();
self
.writer
()
.word
(
pprust
::
literal_to_string
(
token
,
suffix
))
self
.writer
()
.word
(
pprust
::
literal_to_string
(
lit
.node
.to_lit_token
()))
}
pub
fn
print_expr
(
&
mut
self
,
expr
:
&
hir
::
Expr
)
->
io
::
Result
<
()
>
{
...
...
src/librustc/ich/impls_syntax.rs
浏览文件 @
ca2a50fa
...
...
@@ -165,7 +165,6 @@ fn hash_stable<W: StableHasherResult>(&self,
impl_stable_hash_for!
(
struct
::
syntax
::
ast
::
Lit
{
node
,
token
,
suffix
,
span
});
...
...
@@ -288,17 +287,23 @@ fn hash_stable<W: StableHasherResult>(&self,
}
}
impl_stable_hash_for!
(
enum
token
::
Lit
{
Bool
(
val
),
Byte
(
val
),
Char
(
val
),
Err
(
val
),
Integer
(
val
),
Float
(
val
),
Str_
(
val
),
ByteStr
(
val
),
StrRaw
(
val
,
n
),
ByteStrRaw
(
val
,
n
)
impl_stable_hash_for!
(
enum
token
::
LitKind
{
Bool
,
Byte
,
Char
,
Integer
,
Float
,
Str
,
ByteStr
,
StrRaw
(
n
),
ByteStrRaw
(
n
),
Err
});
impl_stable_hash_for!
(
struct
token
::
Lit
{
kind
,
symbol
,
suffix
});
fn
hash_token
<
'a
,
'gcx
,
W
:
StableHasherResult
>
(
...
...
@@ -348,10 +353,7 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
token
::
Token
::
CloseDelim
(
delim_token
)
=>
{
std_hash
::
Hash
::
hash
(
&
delim_token
,
hasher
);
}
token
::
Token
::
Literal
(
lit
,
opt_name
)
=>
{
lit
.hash_stable
(
hcx
,
hasher
);
opt_name
.hash_stable
(
hcx
,
hasher
);
}
token
::
Token
::
Literal
(
lit
)
=>
lit
.hash_stable
(
hcx
,
hasher
),
token
::
Token
::
Ident
(
ident
,
is_raw
)
=>
{
ident
.name
.hash_stable
(
hcx
,
hasher
);
...
...
src/librustdoc/html/highlight.rs
浏览文件 @
ca2a50fa
...
...
@@ -310,17 +310,17 @@ fn write_token<W: Writer>(&mut self,
}
}
token
::
Literal
(
lit
,
_
suf
)
=>
{
match
lit
{
token
::
Literal
(
lit
)
=>
{
match
lit
.kind
{
// Text literals.
token
::
Byte
(
..
)
|
token
::
Char
(
..
)
|
token
::
Err
(
..
)
|
token
::
ByteStr
(
..
)
|
token
::
ByteStrRaw
(
..
)
|
token
::
Str_
(
..
)
|
token
::
StrRaw
(
..
)
=>
Class
::
String
,
token
::
Byte
|
token
::
Char
|
token
::
Err
|
token
::
ByteStr
|
token
::
ByteStrRaw
(
..
)
|
token
::
Str
|
token
::
StrRaw
(
..
)
=>
Class
::
String
,
// Number literals.
token
::
Integer
(
..
)
|
token
::
Float
(
..
)
=>
Class
::
Number
,
token
::
Integer
|
token
::
Float
=>
Class
::
Number
,
token
::
Bool
(
..
)
=>
panic!
(
"literal token contains `Lit::Bool`"
),
token
::
Bool
=>
panic!
(
"literal token contains `Lit::Bool`"
),
}
}
...
...
src/libsyntax/ast.rs
浏览文件 @
ca2a50fa
...
...
@@ -1347,8 +1347,6 @@ pub enum StrStyle {
pub
struct
Lit
{
/// The original literal token as written in source code.
pub
token
:
token
::
Lit
,
/// The original literal suffix as written in source code.
pub
suffix
:
Option
<
Symbol
>
,
/// The "semantic" representation of the literal lowered from the original tokens.
/// Strings are unescaped, hexadecimal forms are eliminated, etc.
/// FIXME: Remove this and only create the semantic representation during lowering to HIR.
...
...
src/libsyntax/diagnostics/plugin.rs
浏览文件 @
ca2a50fa
...
...
@@ -77,8 +77,8 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
},
(
3
,
Some
(
&
TokenTree
::
Token
(
_
,
token
::
Ident
(
ref
code
,
_
))),
Some
(
&
TokenTree
::
Token
(
_
,
token
::
Comma
)),
Some
(
&
TokenTree
::
Token
(
_
,
token
::
Literal
(
token
::
StrRaw
(
description
,
_
),
None
))))
=>
{
(
code
,
Some
(
description
))
Some
(
&
TokenTree
::
Token
(
_
,
token
::
Literal
(
token
::
Lit
{
symbol
,
..
}
))))
=>
{
(
code
,
Some
(
symbol
))
}
_
=>
unreachable!
()
};
...
...
src/libsyntax/parse/lexer/mod.rs
浏览文件 @
ca2a50fa
use
crate
::
ast
::{
self
,
Ident
};
use
crate
::
parse
::{
token
,
ParseSess
};
use
crate
::
parse
::
ParseSess
;
use
crate
::
parse
::
token
::{
self
,
Token
};
use
crate
::
symbol
::
Symbol
;
use
crate
::
parse
::
unescape
;
use
crate
::
parse
::
unescape_error_reporting
::{
emit_unescape_error
,
push_escaped_char
};
...
...
@@ -21,7 +22,7 @@
#[derive(Clone,
Debug)]
pub
struct
TokenAndSpan
{
pub
tok
:
token
::
Token
,
pub
tok
:
Token
,
pub
sp
:
Span
,
}
...
...
@@ -55,7 +56,7 @@ pub struct StringReader<'a> {
/// Stop reading src at this index.
crate
end_src_index
:
usize
,
// cached:
peek_tok
:
token
::
Token
,
peek_tok
:
Token
,
peek_span
:
Span
,
peek_span_src_raw
:
Span
,
fatal_errs
:
Vec
<
DiagnosticBuilder
<
'a
>>
,
...
...
@@ -726,7 +727,7 @@ fn scan_digits(&mut self, real_radix: u32, scan_radix: u32) -> usize {
}
/// Lex a LIT_INTEGER or a LIT_FLOAT
fn
scan_number
(
&
mut
self
,
c
:
char
)
->
token
::
Lit
{
fn
scan_number
(
&
mut
self
,
c
:
char
)
->
(
token
::
LitKind
,
Symbol
)
{
let
mut
base
=
10
;
let
start_bpos
=
self
.pos
;
self
.bump
();
...
...
@@ -753,7 +754,7 @@ fn scan_number(&mut self, c: char) -> token::Lit {
}
_
=>
{
// just a 0
return
token
::
Integer
(
self
.name_from
(
start_bpos
));
return
(
token
::
Integer
,
self
.name_from
(
start_bpos
));
}
}
}
else
if
c
.is_digit
(
10
)
{
...
...
@@ -765,7 +766,7 @@ fn scan_number(&mut self, c: char) -> token::Lit {
if
num_digits
==
0
{
self
.err_span_
(
start_bpos
,
self
.pos
,
"no valid digits found for number"
);
return
token
::
Integer
(
Symbol
::
intern
(
"0"
));
return
(
token
::
Integer
,
Symbol
::
intern
(
"0"
));
}
// might be a float, but don't be greedy if this is actually an
...
...
@@ -783,17 +784,17 @@ fn scan_number(&mut self, c: char) -> token::Lit {
let
pos
=
self
.pos
;
self
.check_float_base
(
start_bpos
,
pos
,
base
);
token
::
Float
(
self
.name_from
(
start_bpos
))
(
token
::
Float
,
self
.name_from
(
start_bpos
))
}
else
{
// it might be a float if it has an exponent
if
self
.ch_is
(
'e'
)
||
self
.ch_is
(
'E'
)
{
self
.scan_float_exponent
();
let
pos
=
self
.pos
;
self
.check_float_base
(
start_bpos
,
pos
,
base
);
return
token
::
Float
(
self
.name_from
(
start_bpos
));
return
(
token
::
Float
,
self
.name_from
(
start_bpos
));
}
// but we certainly have an integer!
token
::
Integer
(
self
.name_from
(
start_bpos
))
(
token
::
Integer
,
self
.name_from
(
start_bpos
))
}
}
...
...
@@ -846,7 +847,7 @@ fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: us
}
}
fn
binop
(
&
mut
self
,
op
:
token
::
BinOpToken
)
->
token
::
Token
{
fn
binop
(
&
mut
self
,
op
:
token
::
BinOpToken
)
->
Token
{
self
.bump
();
if
self
.ch_is
(
'='
)
{
self
.bump
();
...
...
@@ -858,7 +859,7 @@ fn binop(&mut self, op: token::BinOpToken) -> token::Token {
/// Returns the next token from the string, advances the input past that
/// token, and updates the interner
fn
next_token_inner
(
&
mut
self
)
->
Result
<
token
::
Token
,
()
>
{
fn
next_token_inner
(
&
mut
self
)
->
Result
<
Token
,
()
>
{
let
c
=
self
.ch
;
if
ident_start
(
c
)
{
...
...
@@ -912,10 +913,10 @@ fn next_token_inner(&mut self) -> Result<token::Token, ()> {
}
if
is_dec_digit
(
c
)
{
let
num
=
self
.scan_number
(
c
.unwrap
());
let
(
kind
,
symbol
)
=
self
.scan_number
(
c
.unwrap
());
let
suffix
=
self
.scan_optional_raw_name
();
debug!
(
"next_token_inner: scanned number {:?}, {:?}
"
,
num
,
suffix
);
return
Ok
(
token
::
Literal
(
num
,
suffix
));
debug!
(
"next_token_inner: scanned number {:?}, {:?}
, {:?}"
,
kind
,
symbol
,
suffix
);
return
Ok
(
Token
::
lit
(
kind
,
symbol
,
suffix
));
}
match
c
.expect
(
"next_token_inner called at EOF"
)
{
...
...
@@ -1073,10 +1074,10 @@ fn next_token_inner(&mut self) -> Result<token::Token, ()> {
// lifetimes shouldn't end with a single quote
// if we find one, then this is an invalid character literal
if
self
.ch_is
(
'\''
)
{
let
id
=
self
.name_from
(
start
);
let
symbol
=
self
.name_from
(
start
);
self
.bump
();
self
.validate_char_escape
(
start_with_quote
);
return
Ok
(
token
::
Literal
(
token
::
Char
(
id
),
None
))
return
Ok
(
Token
::
lit
(
token
::
Char
,
symbol
,
None
));
}
// Include the leading `'` in the real identifier, for macro
...
...
@@ -1098,43 +1099,43 @@ fn next_token_inner(&mut self) -> Result<token::Token, ()> {
return
Ok
(
token
::
Lifetime
(
ident
));
}
let
msg
=
"unterminated character literal"
;
let
id
=
self
.scan_single_quoted_string
(
start_with_quote
,
msg
);
let
symbol
=
self
.scan_single_quoted_string
(
start_with_quote
,
msg
);
self
.validate_char_escape
(
start_with_quote
);
let
suffix
=
self
.scan_optional_raw_name
();
Ok
(
token
::
Literal
(
token
::
Char
(
id
)
,
suffix
))
Ok
(
Token
::
lit
(
token
::
Char
,
symbol
,
suffix
))
}
'b'
=>
{
self
.bump
();
let
lit
=
match
self
.ch
{
let
(
kind
,
symbol
)
=
match
self
.ch
{
Some
(
'\''
)
=>
{
let
start_with_quote
=
self
.pos
;
self
.bump
();
let
msg
=
"unterminated byte constant"
;
let
id
=
self
.scan_single_quoted_string
(
start_with_quote
,
msg
);
let
symbol
=
self
.scan_single_quoted_string
(
start_with_quote
,
msg
);
self
.validate_byte_escape
(
start_with_quote
);
token
::
Byte
(
id
)
(
token
::
Byte
,
symbol
)
},
Some
(
'"'
)
=>
{
let
start_with_quote
=
self
.pos
;
let
msg
=
"unterminated double quote byte string"
;
let
id
=
self
.scan_double_quoted_string
(
msg
);
let
symbol
=
self
.scan_double_quoted_string
(
msg
);
self
.validate_byte_str_escape
(
start_with_quote
);
token
::
ByteStr
(
id
)
(
token
::
ByteStr
,
symbol
)
},
Some
(
'r'
)
=>
self
.scan_raw_byte_string
(),
_
=>
unreachable!
(),
// Should have been a token::Ident above.
};
let
suffix
=
self
.scan_optional_raw_name
();
Ok
(
token
::
Literal
(
lit
,
suffix
))
Ok
(
Token
::
lit
(
kind
,
symbol
,
suffix
))
}
'"'
=>
{
let
start_with_quote
=
self
.pos
;
let
msg
=
"unterminated double quote string"
;
let
id
=
self
.scan_double_quoted_string
(
msg
);
let
symbol
=
self
.scan_double_quoted_string
(
msg
);
self
.validate_str_escape
(
start_with_quote
);
let
suffix
=
self
.scan_optional_raw_name
();
Ok
(
token
::
Literal
(
token
::
Str_
(
id
)
,
suffix
))
Ok
(
Token
::
lit
(
token
::
Str
,
symbol
,
suffix
))
}
'r'
=>
{
let
start_bpos
=
self
.pos
;
...
...
@@ -1205,14 +1206,14 @@ fn next_token_inner(&mut self) -> Result<token::Token, ()> {
}
self
.bump
();
let
id
=
if
valid
{
let
symbol
=
if
valid
{
self
.name_from_to
(
content_start_bpos
,
content_end_bpos
)
}
else
{
Symbol
::
intern
(
"??"
)
};
let
suffix
=
self
.scan_optional_raw_name
();
Ok
(
token
::
Literal
(
token
::
StrRaw
(
id
,
hash_count
)
,
suffix
))
Ok
(
Token
::
lit
(
token
::
StrRaw
(
hash_count
),
symbol
,
suffix
))
}
'-'
=>
{
if
self
.nextch_is
(
'>'
)
{
...
...
@@ -1366,7 +1367,7 @@ fn scan_double_quoted_string(&mut self, unterminated_msg: &str) -> ast::Name {
id
}
fn
scan_raw_byte_string
(
&
mut
self
)
->
token
::
Lit
{
fn
scan_raw_byte_string
(
&
mut
self
)
->
(
token
::
LitKind
,
Symbol
)
{
let
start_bpos
=
self
.pos
;
self
.bump
();
let
mut
hash_count
=
0
;
...
...
@@ -1423,7 +1424,7 @@ fn scan_raw_byte_string(&mut self) -> token::Lit {
self
.bump
();
token
::
ByteStrRaw
(
self
.name_from_to
(
content_start_bpos
,
content_end_bpos
),
hash_count
)
(
token
::
ByteStrRaw
(
hash_count
),
self
.name_from_to
(
content_start_bpos
,
content_end_bpos
)
)
}
fn
validate_char_escape
(
&
self
,
start_with_quote
:
BytePos
)
{
...
...
@@ -1637,15 +1638,19 @@ fn t1() {
// check that the given reader produces the desired stream
// of tokens (stop checking after exhausting the expected vec)
fn
check_tokenization
(
mut
string_reader
:
StringReader
<
'_
>
,
expected
:
Vec
<
token
::
Token
>
)
{
fn
check_tokenization
(
mut
string_reader
:
StringReader
<
'_
>
,
expected
:
Vec
<
Token
>
)
{
for
expected_tok
in
&
expected
{
assert_eq!
(
&
string_reader
.next_token
()
.tok
,
expected_tok
);
}
}
// make the identifier by looking up the string in the interner
fn
mk_ident
(
id
:
&
str
)
->
token
::
Token
{
token
::
Token
::
from_ast_ident
(
Ident
::
from_str
(
id
))
fn
mk_ident
(
id
:
&
str
)
->
Token
{
Token
::
from_ast_ident
(
Ident
::
from_str
(
id
))
}
fn
mk_lit
(
kind
:
token
::
LitKind
,
symbol
:
&
str
,
suffix
:
Option
<&
str
>
)
->
Token
{
Token
::
lit
(
kind
,
Symbol
::
intern
(
symbol
),
suffix
.map
(
Symbol
::
intern
))
}
#[test]
...
...
@@ -1694,7 +1699,7 @@ fn character_a() {
let
sm
=
Lrc
::
new
(
SourceMap
::
new
(
FilePathMapping
::
empty
()));
let
sh
=
mk_sess
(
sm
.clone
());
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"'a'"
.to_string
())
.next_token
()
.tok
,
token
::
Literal
(
token
::
Char
(
Symbol
::
intern
(
"a"
))
,
None
));
mk_lit
(
token
::
Char
,
"a"
,
None
));
})
}
...
...
@@ -1704,7 +1709,7 @@ fn character_space() {
let
sm
=
Lrc
::
new
(
SourceMap
::
new
(
FilePathMapping
::
empty
()));
let
sh
=
mk_sess
(
sm
.clone
());
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"' '"
.to_string
())
.next_token
()
.tok
,
token
::
Literal
(
token
::
Char
(
Symbol
::
intern
(
" "
))
,
None
));
mk_lit
(
token
::
Char
,
" "
,
None
));
})
}
...
...
@@ -1714,7 +1719,7 @@ fn character_escaped() {
let
sm
=
Lrc
::
new
(
SourceMap
::
new
(
FilePathMapping
::
empty
()));
let
sh
=
mk_sess
(
sm
.clone
());
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"'
\\
n'"
.to_string
())
.next_token
()
.tok
,
token
::
Literal
(
token
::
Char
(
Symbol
::
intern
(
"
\\
n"
))
,
None
));
mk_lit
(
token
::
Char
,
"
\\
n"
,
None
));
})
}
...
...
@@ -1724,7 +1729,7 @@ fn lifetime_name() {
let
sm
=
Lrc
::
new
(
SourceMap
::
new
(
FilePathMapping
::
empty
()));
let
sh
=
mk_sess
(
sm
.clone
());
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"'abc"
.to_string
())
.next_token
()
.tok
,
token
::
Lifetime
(
Ident
::
from_str
(
"'abc"
)));
token
::
Lifetime
(
Ident
::
from_str
(
"'abc"
)));
})
}
...
...
@@ -1733,10 +1738,8 @@ fn raw_string() {
with_default_globals
(||
{
let
sm
=
Lrc
::
new
(
SourceMap
::
new
(
FilePathMapping
::
empty
()));
let
sh
=
mk_sess
(
sm
.clone
());
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"r###
\"\"
#a
\\
b
\x00
c
\"\"
###"
.to_string
())
.next_token
()
.tok
,
token
::
Literal
(
token
::
StrRaw
(
Symbol
::
intern
(
"
\"
#a
\\
b
\x00
c
\"
"
),
3
),
None
));
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"r###
\"\"
#a
\\
b
\x00
c
\"\"
###"
.to_string
())
.next_token
()
.tok
,
mk_lit
(
token
::
StrRaw
(
3
),
"
\"
#a
\\
b
\x00
c
\"
"
,
None
));
})
}
...
...
@@ -1748,18 +1751,16 @@ fn literal_suffixes() {
macro_rules!
test
{
(
$input
:
expr
,
$tok_type
:
ident
,
$tok_contents
:
expr
)
=>
{{
assert_eq!
(
setup
(
&
sm
,
&
sh
,
format!
(
"{}suffix"
,
$input
))
.next_token
()
.tok
,
token
::
Literal
(
token
::
$tok_type
(
Symbol
::
intern
(
$tok_contents
)),
Some
(
Symbol
::
intern
(
"suffix"
))));
mk_lit
(
token
::
$tok_type
,
$tok_contents
,
Some
(
"suffix"
)));
// with a whitespace separator:
assert_eq!
(
setup
(
&
sm
,
&
sh
,
format!
(
"{} suffix"
,
$input
))
.next_token
()
.tok
,
token
::
Literal
(
token
::
$tok_type
(
Symbol
::
intern
(
$tok_contents
)),
None
));
mk_lit
(
token
::
$tok_type
,
$tok_contents
,
None
));
}}
}
test!
(
"'a'"
,
Char
,
"a"
);
test!
(
"b'a'"
,
Byte
,
"a"
);
test!
(
"
\"
a
\"
"
,
Str
_
,
"a"
);
test!
(
"
\"
a
\"
"
,
Str
,
"a"
);
test!
(
"b
\"
a
\"
"
,
ByteStr
,
"a"
);
test!
(
"1234"
,
Integer
,
"1234"
);
test!
(
"0b101"
,
Integer
,
"0b101"
);
...
...
@@ -1768,14 +1769,11 @@ fn literal_suffixes() {
test!
(
"1.0e10"
,
Float
,
"1.0e10"
);
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"2us"
.to_string
())
.next_token
()
.tok
,
token
::
Literal
(
token
::
Integer
(
Symbol
::
intern
(
"2"
)),
Some
(
Symbol
::
intern
(
"us"
))));
mk_lit
(
token
::
Integer
,
"2"
,
Some
(
"us"
)));
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"r###
\"
raw
\"
###suffix"
.to_string
())
.next_token
()
.tok
,
token
::
Literal
(
token
::
StrRaw
(
Symbol
::
intern
(
"raw"
),
3
),
Some
(
Symbol
::
intern
(
"suffix"
))));
mk_lit
(
token
::
StrRaw
(
3
),
"raw"
,
Some
(
"suffix"
)));
assert_eq!
(
setup
(
&
sm
,
&
sh
,
"br###
\"
raw
\"
###suffix"
.to_string
())
.next_token
()
.tok
,
token
::
Literal
(
token
::
ByteStrRaw
(
Symbol
::
intern
(
"raw"
),
3
),
Some
(
Symbol
::
intern
(
"suffix"
))));
mk_lit
(
token
::
ByteStrRaw
(
3
),
"raw"
,
Some
(
"suffix"
)));
})
}
...
...
@@ -1796,8 +1794,7 @@ fn nested_block_comments() {
token
::
Comment
=>
{}
_
=>
panic!
(
"expected a comment!"
),
}
assert_eq!
(
lexer
.next_token
()
.tok
,
token
::
Literal
(
token
::
Char
(
Symbol
::
intern
(
"a"
)),
None
));
assert_eq!
(
lexer
.next_token
()
.tok
,
mk_lit
(
token
::
Char
,
"a"
,
None
));
})
}
...
...
src/libsyntax/parse/literal.rs
浏览文件 @
ca2a50fa
...
...
@@ -3,7 +3,7 @@
use
crate
::
ast
::{
self
,
Ident
,
Lit
,
LitKind
};
use
crate
::
parse
::
parser
::
Parser
;
use
crate
::
parse
::
PResult
;
use
crate
::
parse
::
token
;
use
crate
::
parse
::
token
::{
self
,
Token
}
;
use
crate
::
parse
::
unescape
::{
unescape_str
,
unescape_char
,
unescape_byte_str
,
unescape_byte
};
use
crate
::
print
::
pprust
;
use
crate
::
symbol
::{
kw
,
Symbol
};
...
...
@@ -27,14 +27,21 @@
}
impl
LitError
{
crate
fn
report
(
&
self
,
diag
:
&
Handler
,
lit
:
token
::
Lit
,
suf
:
Option
<
Symbol
>
,
span
:
Span
)
{
crate
fn
report
(
&
self
,
diag
:
&
Handler
,
token
::
Lit
{
kind
,
suffix
,
..
}:
token
::
Lit
,
span
:
Span
,
)
{
match
*
self
{
LitError
::
NotLiteral
|
LitError
::
LexerError
=>
{}
LitError
::
InvalidSuffix
=>
{
expect_no_suffix
(
diag
,
span
,
&
format!
(
"{} {}"
,
lit
.article
(),
lit
.descr
()),
suf
);
expect_no_suffix
(
diag
,
span
,
&
format!
(
"{} {}"
,
kind
.article
(),
kind
.descr
()),
suffix
);
}
LitError
::
InvalidIntSuffix
=>
{
let
suf
=
suf
.expect
(
"suffix error with no suffix"
)
.as_str
();
let
suf
=
suf
fix
.expect
(
"suffix error with no suffix"
)
.as_str
();
if
looks_like_width_suffix
(
&
[
'i'
,
'u'
],
&
suf
)
{
// If it looks like a width, try to be helpful.
let
msg
=
format!
(
"invalid width `{}` for integer literal"
,
&
suf
[
1
..
]);
...
...
@@ -50,7 +57,7 @@ impl LitError {
}
}
LitError
::
InvalidFloatSuffix
=>
{
let
suf
=
suf
.expect
(
"suffix error with no suffix"
)
.as_str
();
let
suf
=
suf
fix
.expect
(
"suffix error with no suffix"
)
.as_str
();
if
looks_like_width_suffix
(
&
[
'f'
],
&
suf
)
{
// If it looks like a width, try to be helpful.
let
msg
=
format!
(
"invalid width `{}` for float literal"
,
&
suf
[
1
..
]);
...
...
@@ -84,43 +91,42 @@ impl LitKind {
/// If diagnostic handler is passed, always returns `Some`,
/// possibly after reporting non-fatal errors and recovery.
fn
from_lit_token
(
lit
:
token
::
Lit
,
suf
:
Option
<
Symbol
>
,
token
::
Lit
{
kind
,
symbol
,
suffix
}:
token
::
Lit
,
)
->
Result
<
LitKind
,
LitError
>
{
if
suf
.is_some
()
&&
!
lit
.may_have_suffix
()
{
if
suf
fix
.is_some
()
&&
!
kind
.may_have_suffix
()
{
return
Err
(
LitError
::
InvalidSuffix
);
}
Ok
(
match
lit
{
token
::
Bool
(
i
)
=>
{
assert
!
(
i
==
kw
::
True
||
i
==
kw
::
False
);
LitKind
::
Bool
(
i
==
kw
::
True
)
Ok
(
match
kind
{
token
::
Bool
=>
{
assert
!
(
symbol
==
kw
::
True
||
symbol
==
kw
::
False
);
LitKind
::
Bool
(
symbol
==
kw
::
True
)
}
token
::
Byte
(
i
)
=>
{
match
unescape_byte
(
&
i
.as_str
())
{
token
::
Byte
=>
{
match
unescape_byte
(
&
symbol
.as_str
())
{
Ok
(
c
)
=>
LitKind
::
Byte
(
c
),
Err
(
_
)
=>
LitKind
::
Err
(
i
),
Err
(
_
)
=>
return
Err
(
LitError
::
LexerError
),
}
},
token
::
Char
(
i
)
=>
{
match
unescape_char
(
&
i
.as_str
())
{
token
::
Char
=>
{
match
unescape_char
(
&
symbol
.as_str
())
{
Ok
(
c
)
=>
LitKind
::
Char
(
c
),
Err
(
_
)
=>
return
Err
(
LitError
::
LexerError
),
}
},
token
::
Err
(
i
)
=>
LitKind
::
Err
(
i
),
// There are some valid suffixes for integer and float literals,
// so all the handling is done internally.
token
::
Integer
(
s
)
=>
return
integer_lit
(
s
,
suf
),
token
::
Float
(
s
)
=>
return
float_lit
(
s
,
suf
),
token
::
Integer
=>
return
integer_lit
(
symbol
,
suffix
),
token
::
Float
=>
return
float_lit
(
symbol
,
suffix
),
token
::
Str
_
(
mut
sym
)
=>
{
token
::
Str
=>
{
// If there are no characters requiring special treatment we can
// reuse the symbol from the token. Otherwise, we must generate a
// new symbol because the string in the LitKind is different to the
// string in the token.
let
mut
error
=
None
;
let
mut
sym
=
symbol
;
let
s
=
&
sym
.as_str
();
if
s
.as_bytes
()
.iter
()
.any
(|
&
c
|
c
==
b
'\\'
||
c
==
b
'\r'
)
{
let
mut
buf
=
String
::
with_capacity
(
s
.len
());
...
...
@@ -138,16 +144,17 @@ fn from_lit_token(
LitKind
::
Str
(
sym
,
ast
::
StrStyle
::
Cooked
)
}
token
::
StrRaw
(
mut
sym
,
n
)
=>
{
token
::
StrRaw
(
n
)
=>
{
// Ditto.
let
mut
sym
=
symbol
;
let
s
=
&
sym
.as_str
();
if
s
.contains
(
'\r'
)
{
sym
=
Symbol
::
intern
(
&
raw_str_lit
(
s
));
}
LitKind
::
Str
(
sym
,
ast
::
StrStyle
::
Raw
(
n
))
}
token
::
ByteStr
(
i
)
=>
{
let
s
=
&
i
.as_str
();
token
::
ByteStr
=>
{
let
s
=
&
symbol
.as_str
();
let
mut
buf
=
Vec
::
with_capacity
(
s
.len
());
let
mut
error
=
None
;
unescape_byte_str
(
s
,
&
mut
|
_
,
unescaped_byte
|
{
...
...
@@ -162,36 +169,37 @@ fn from_lit_token(
buf
.shrink_to_fit
();
LitKind
::
ByteStr
(
Lrc
::
new
(
buf
))
}
token
::
ByteStrRaw
(
i
,
_
)
=>
{
LitKind
::
ByteStr
(
Lrc
::
new
(
i
.to_string
()
.into_bytes
()))
token
::
ByteStrRaw
(
_
)
=>
{
LitKind
::
ByteStr
(
Lrc
::
new
(
symbol
.to_string
()
.into_bytes
()))
}
token
::
Err
=>
LitKind
::
Err
(
symbol
),
})
}
/// Attempts to recover a token from semantic literal.
/// This function is used when the original token doesn't exist (e.g. the literal is created
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
pub
fn
to_lit_token
(
&
self
)
->
(
token
::
Lit
,
Option
<
Symbol
>
)
{
match
*
self
{
pub
fn
to_lit_token
(
&
self
)
->
token
::
Lit
{
let
(
kind
,
symbol
,
suffix
)
=
match
*
self
{
LitKind
::
Str
(
string
,
ast
::
StrStyle
::
Cooked
)
=>
{
let
escaped
=
string
.as_str
()
.escape_default
()
.to_string
();
(
token
::
Lit
::
Str_
(
Symbol
::
intern
(
&
escaped
)
),
None
)
(
token
::
Str
,
Symbol
::
intern
(
&
escaped
),
None
)
}
LitKind
::
Str
(
string
,
ast
::
StrStyle
::
Raw
(
n
))
=>
{
(
token
::
Lit
::
StrRaw
(
string
,
n
)
,
None
)
(
token
::
StrRaw
(
n
),
string
,
None
)
}
LitKind
::
ByteStr
(
ref
bytes
)
=>
{
let
string
=
bytes
.iter
()
.cloned
()
.flat_map
(
ascii
::
escape_default
)
.map
(
Into
::
<
char
>
::
into
)
.collect
::
<
String
>
();
(
token
::
Lit
::
ByteStr
(
Symbol
::
intern
(
&
string
)
),
None
)
(
token
::
ByteStr
,
Symbol
::
intern
(
&
string
),
None
)
}
LitKind
::
Byte
(
byte
)
=>
{
let
string
:
String
=
ascii
::
escape_default
(
byte
)
.map
(
Into
::
<
char
>
::
into
)
.collect
();
(
token
::
Lit
::
Byte
(
Symbol
::
intern
(
&
string
)
),
None
)
(
token
::
Byte
,
Symbol
::
intern
(
&
string
),
None
)
}
LitKind
::
Char
(
ch
)
=>
{
let
string
:
String
=
ch
.escape_default
()
.map
(
Into
::
<
char
>
::
into
)
.collect
();
(
token
::
Lit
::
Char
(
Symbol
::
intern
(
&
string
)
),
None
)
(
token
::
Char
,
Symbol
::
intern
(
&
string
),
None
)
}
LitKind
::
Int
(
n
,
ty
)
=>
{
let
suffix
=
match
ty
{
...
...
@@ -199,29 +207,33 @@ pub fn to_lit_token(&self) -> (token::Lit, Option<Symbol>) {
ast
::
LitIntType
::
Signed
(
ty
)
=>
Some
(
Symbol
::
intern
(
ty
.ty_to_string
())),
ast
::
LitIntType
::
Unsuffixed
=>
None
,
};
(
token
::
Lit
::
Integer
(
Symbol
::
intern
(
&
n
.to_string
()
)),
suffix
)
(
token
::
Integer
,
Symbol
::
intern
(
&
n
.to_string
(
)),
suffix
)
}
LitKind
::
Float
(
symbol
,
ty
)
=>
{
(
token
::
Lit
::
Float
(
symbol
),
Some
(
Symbol
::
intern
(
ty
.ty_to_string
())))
(
token
::
Float
,
symbol
,
Some
(
Symbol
::
intern
(
ty
.ty_to_string
())))
}
LitKind
::
FloatUnsuffixed
(
symbol
)
=>
{
(
token
::
Float
,
symbol
,
None
)
}
LitKind
::
FloatUnsuffixed
(
symbol
)
=>
(
token
::
Lit
::
Float
(
symbol
),
None
),
LitKind
::
Bool
(
value
)
=>
{
let
kw
=
if
value
{
kw
::
True
}
else
{
kw
::
False
};
(
token
::
Lit
::
Bool
(
kw
)
,
None
)
let
symbol
=
if
value
{
kw
::
True
}
else
{
kw
::
False
};
(
token
::
Bool
,
symbol
,
None
)
}
LitKind
::
Err
(
val
)
=>
(
token
::
Lit
::
Err
(
val
),
None
),
}
LitKind
::
Err
(
symbol
)
=>
{
(
token
::
Err
,
symbol
,
None
)
}
};
token
::
Lit
::
new
(
kind
,
symbol
,
suffix
)
}
}
impl
Lit
{
fn
from_lit_token
(
token
:
token
::
Lit
,
suffix
:
Option
<
Symbol
>
,
span
:
Span
,
)
->
Result
<
Lit
,
LitError
>
{
let
node
=
LitKind
::
from_lit_token
(
token
,
suffix
)
?
;
Ok
(
Lit
{
node
,
token
,
suffix
,
span
})
Ok
(
Lit
{
token
,
node
:
LitKind
::
from_lit_token
(
token
)
?
,
span
})
}
/// Converts literal token with a suffix into an AST literal.
...
...
@@ -232,11 +244,11 @@ fn from_lit_token(
token
:
&
token
::
Token
,
span
:
Span
,
)
->
Result
<
Lit
,
LitError
>
{
let
(
lit
,
suf
)
=
match
*
token
{
let
lit
=
match
*
token
{
token
::
Ident
(
ident
,
false
)
if
ident
.name
==
kw
::
True
||
ident
.name
==
kw
::
False
=>
(
token
::
Bool
(
ident
.name
)
,
None
),
token
::
Literal
(
token
,
suffix
)
=>
(
token
,
suffix
)
,
token
::
Lit
::
new
(
token
::
Bool
,
ident
.name
,
None
),
token
::
Literal
(
lit
)
=>
lit
,
token
::
Interpolated
(
ref
nt
)
=>
{
if
let
token
::
NtExpr
(
expr
)
|
token
::
NtLiteral
(
expr
)
=
&**
nt
{
if
let
ast
::
ExprKind
::
Lit
(
lit
)
=
&
expr
.node
{
...
...
@@ -248,22 +260,21 @@ fn from_lit_token(
_
=>
return
Err
(
LitError
::
NotLiteral
)
};
Lit
::
from_lit_token
(
lit
,
s
uf
,
s
pan
)
Lit
::
from_lit_token
(
lit
,
span
)
}
/// Attempts to recover an AST literal from semantic literal.
/// This function is used when the original token doesn't exist (e.g. the literal is created
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
pub
fn
from_lit_kind
(
node
:
LitKind
,
span
:
Span
)
->
Lit
{
let
(
token
,
suffix
)
=
node
.to_lit_token
();
Lit
{
node
,
token
,
suffix
,
span
}
Lit
{
token
:
node
.to_lit_token
(),
node
,
span
}
}
/// Losslessly convert an AST literal into a token stream.
crate
fn
tokens
(
&
self
)
->
TokenStream
{
let
token
=
match
self
.token
{
token
::
Bool
(
symbol
)
=>
token
::
Ident
(
Ident
::
new
(
symbol
,
self
.span
),
false
),
token
=>
token
::
Literal
(
token
,
self
.suffix
),
let
token
=
match
self
.token
.kind
{
token
::
Bool
=>
token
::
Ident
(
Ident
::
new
(
self
.token.
symbol
,
self
.span
),
false
),
_
=>
token
::
Literal
(
self
.token
),
};
TokenTree
::
Token
(
self
.span
,
token
)
.into
()
}
...
...
@@ -276,11 +287,11 @@ impl<'a> Parser<'a> {
if
self
.token
==
token
::
Dot
{
// Attempt to recover `.4` as `0.4`.
recovered
=
self
.look_ahead
(
1
,
|
t
|
{
if
let
token
::
Literal
(
token
::
Integer
(
val
),
suf
)
=
*
t
{
if
let
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
Integer
,
symbol
,
suffix
}
)
=
*
t
{
let
next_span
=
self
.look_ahead_span
(
1
);
if
self
.span
.hi
()
==
next_span
.lo
()
{
let
s
ym
=
String
::
from
(
"0."
)
+
&
va
l
.as_str
();
let
token
=
token
::
Literal
(
token
::
Float
(
Symbol
::
intern
(
&
sym
)),
suf
);
let
s
=
String
::
from
(
"0."
)
+
&
symbo
l
.as_str
();
let
token
=
Token
::
lit
(
token
::
Float
,
Symbol
::
intern
(
&
s
),
suffix
);
return
Some
((
token
,
self
.span
.to
(
next_span
)));
}
}
...
...
@@ -313,10 +324,11 @@ impl<'a> Parser<'a> {
return
Err
(
self
.span_fatal
(
span
,
&
msg
));
}
Err
(
err
)
=>
{
let
(
lit
,
suf
)
=
token
.expect_lit
();
let
lit
=
token
.expect_lit
();
self
.bump
();
err
.report
(
&
self
.sess.span_diagnostic
,
lit
,
suf
,
span
);
return
Ok
(
Lit
::
from_lit_token
(
token
::
Err
(
lit
.symbol
()),
suf
,
span
)
.ok
()
.unwrap
());
err
.report
(
&
self
.sess.span_diagnostic
,
lit
,
span
);
let
lit
=
token
::
Lit
::
new
(
token
::
Err
,
lit
.symbol
,
lit
.suffix
);
return
Ok
(
Lit
::
from_lit_token
(
lit
,
span
)
.ok
()
.unwrap
());
}
}
}
...
...
src/libsyntax/parse/parser.rs
浏览文件 @
ca2a50fa
...
...
@@ -352,10 +352,12 @@ fn next_desugared(&mut self) -> TokenAndSpan {
let
body
=
TokenTree
::
Delimited
(
delim_span
,
token
::
Bracket
,
[
TokenTree
::
Token
(
sp
,
token
::
Ident
(
ast
::
Ident
::
with_empty_ctxt
(
sym
::
doc
),
false
)),
TokenTree
::
Token
(
sp
,
token
::
Eq
),
TokenTree
::
Token
(
sp
,
token
::
Literal
(
token
::
StrRaw
(
Symbol
::
intern
(
&
stripped
),
num_of_hashes
),
None
))
[
TokenTree
::
Token
(
sp
,
token
::
Ident
(
ast
::
Ident
::
with_empty_ctxt
(
sym
::
doc
),
false
)),
TokenTree
::
Token
(
sp
,
token
::
Eq
),
TokenTree
::
Token
(
sp
,
token
::
Token
::
lit
(
token
::
StrRaw
(
num_of_hashes
),
Symbol
::
intern
(
&
stripped
),
None
)),
]
.iter
()
.cloned
()
.collect
::
<
TokenStream
>
()
.into
(),
);
...
...
@@ -2241,10 +2243,10 @@ fn parse_mutability(&mut self) -> Mutability {
}
fn
parse_field_name
(
&
mut
self
)
->
PResult
<
'a
,
Ident
>
{
if
let
token
::
Literal
(
token
::
Integer
(
name
),
suffix
)
=
self
.token
{
if
let
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
Integer
,
symbol
,
suffix
}
)
=
self
.token
{
self
.expect_no_suffix
(
self
.span
,
"a tuple index"
,
suffix
);
self
.bump
();
Ok
(
Ident
::
new
(
name
,
self
.prev_span
))
Ok
(
Ident
::
new
(
symbol
,
self
.prev_span
))
}
else
{
self
.parse_ident_common
(
false
)
}
...
...
@@ -3045,19 +3047,19 @@ fn parse_dot_or_call_expr_with_(&mut self, e0: P<Expr>, lo: Span) -> PResult<'a,
token
::
Ident
(
..
)
=>
{
e
=
self
.parse_dot_suffix
(
e
,
lo
)
?
;
}
token
::
Literal
(
token
::
Integer
(
name
),
suffix
)
=>
{
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
Integer
,
symbol
,
suffix
}
)
=>
{
let
span
=
self
.span
;
self
.bump
();
let
field
=
ExprKind
::
Field
(
e
,
Ident
::
new
(
name
,
span
));
let
field
=
ExprKind
::
Field
(
e
,
Ident
::
new
(
symbol
,
span
));
e
=
self
.mk_expr
(
lo
.to
(
span
),
field
,
ThinVec
::
new
());
self
.expect_no_suffix
(
span
,
"a tuple index"
,
suffix
);
}
token
::
Literal
(
token
::
Float
(
n
),
_
suf
)
=>
{
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
Float
,
symbol
,
..
}
)
=>
{
self
.bump
();
let
fstr
=
n
.as_str
();
let
m
ut
err
=
self
.diagnostic
()
.struct_span_err
(
self
.prev_span
,
&
format!
(
"unexpected token: `{}`"
,
n
)
);
let
fstr
=
symbol
.as_str
();
let
m
sg
=
format!
(
"unexpected token: `{}`"
,
symbol
);
let
mut
err
=
self
.diagnostic
()
.struct_span_err
(
self
.prev_span
,
&
msg
);
err
.span_label
(
self
.prev_span
,
"unexpected token"
);
if
fstr
.chars
()
.all
(|
x
|
"0123456789."
.contains
(
x
))
{
let
float
=
match
fstr
.parse
::
<
f64
>
()
.ok
()
{
...
...
@@ -7557,11 +7559,12 @@ fn parse_item_enum(&mut self) -> PResult<'a, ItemInfo> {
/// the `extern` keyword, if one is found.
fn
parse_opt_abi
(
&
mut
self
)
->
PResult
<
'a
,
Option
<
Abi
>>
{
match
self
.token
{
token
::
Literal
(
token
::
Str_
(
s
),
suf
)
|
token
::
Literal
(
token
::
StrRaw
(
s
,
_
),
suf
)
=>
{
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
Str
,
symbol
,
suffix
})
|
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
StrRaw
(
..
),
symbol
,
suffix
})
=>
{
let
sp
=
self
.span
;
self
.expect_no_suffix
(
sp
,
"an ABI spec"
,
suf
);
self
.expect_no_suffix
(
sp
,
"an ABI spec"
,
suf
fix
);
self
.bump
();
match
abi
::
lookup
(
&
s
.as_str
())
{
match
abi
::
lookup
(
&
s
ymbol
.as_str
())
{
Some
(
abi
)
=>
Ok
(
Some
(
abi
)),
None
=>
{
let
prev_span
=
self
.prev_span
;
...
...
@@ -7570,7 +7573,7 @@ fn parse_opt_abi(&mut self) -> PResult<'a, Option<Abi>> {
prev_span
,
E0703
,
"invalid ABI: found `{}`"
,
s
);
s
ymbol
);
err
.span_label
(
prev_span
,
"invalid ABI"
);
err
.help
(
&
format!
(
"valid ABIs: {}"
,
abi
::
all_names
()
.join
(
", "
)));
err
.emit
();
...
...
@@ -8370,8 +8373,10 @@ pub fn parse_crate_mod(&mut self) -> PResult<'a, Crate> {
pub
fn
parse_optional_str
(
&
mut
self
)
->
Option
<
(
Symbol
,
ast
::
StrStyle
,
Option
<
ast
::
Name
>
)
>
{
let
ret
=
match
self
.token
{
token
::
Literal
(
token
::
Str_
(
s
),
suf
)
=>
(
s
,
ast
::
StrStyle
::
Cooked
,
suf
),
token
::
Literal
(
token
::
StrRaw
(
s
,
n
),
suf
)
=>
(
s
,
ast
::
StrStyle
::
Raw
(
n
),
suf
),
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
Str
,
symbol
,
suffix
})
=>
(
symbol
,
ast
::
StrStyle
::
Cooked
,
suffix
),
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
StrRaw
(
n
),
symbol
,
suffix
})
=>
(
symbol
,
ast
::
StrStyle
::
Raw
(
n
),
suffix
),
_
=>
return
None
};
self
.bump
();
...
...
src/libsyntax/parse/token.rs
浏览文件 @
ca2a50fa
pub
use
BinOpToken
::
*
;
pub
use
Nonterminal
::
*
;
pub
use
DelimToken
::
*
;
pub
use
Lit
::
*
;
pub
use
Lit
Kind
::
*
;
pub
use
Token
::
*
;
use
crate
::
ast
::{
self
};
...
...
@@ -59,59 +59,62 @@ pub fn is_empty(self) -> bool {
}
}
#[derive(Clone,
PartialEq,
RustcEncodable,
RustcDecodable,
Hash,
Debug,
Copy
)]
pub
enum
Lit
{
Bool
(
ast
::
Name
)
,
// AST only, must never appear in a `Token`
Byte
(
ast
::
Name
)
,
Char
(
ast
::
Name
)
,
Err
(
ast
::
Name
)
,
Integer
(
ast
::
Name
)
,
Float
(
ast
::
Name
)
,
Str
_
(
ast
::
Name
),
StrRaw
(
ast
::
Name
,
u16
),
/* raw str delimited by n hash symbols */
ByteStr
(
ast
::
Name
),
ByteStrRaw
(
ast
::
Name
,
u16
),
/* raw byte str delimited by n hash symbols */
#[derive(Clone,
Copy,
PartialEq,
RustcEncodable,
RustcDecodable,
Debug
)]
pub
enum
Lit
Kind
{
Bool
,
// AST only, must never appear in a `Token`
Byte
,
Char
,
Integer
,
Float
,
Str
,
Str
Raw
(
u16
),
// raw string delimited by `n` hash symbols
ByteStr
,
ByteStr
Raw
(
u16
),
// raw byte string delimited by `n` hash symbols
Err
,
}
#[cfg(target_arch
=
"x86_64"
)]
static_assert_size!
(
Lit
,
8
);
impl
Lit
{
crate
fn
symbol
(
&
self
)
->
Symbol
{
match
*
self
{
Bool
(
s
)
|
Byte
(
s
)
|
Char
(
s
)
|
Integer
(
s
)
|
Float
(
s
)
|
Err
(
s
)
|
Str_
(
s
)
|
StrRaw
(
s
,
_
)
|
ByteStr
(
s
)
|
ByteStrRaw
(
s
,
_
)
=>
s
}
}
#[derive(Clone,
Copy,
PartialEq,
RustcEncodable,
RustcDecodable,
Debug)]
pub
struct
Lit
{
pub
kind
:
LitKind
,
pub
symbol
:
Symbol
,
pub
suffix
:
Option
<
Symbol
>
,
}
crate
fn
article
(
&
self
)
->
&
'static
str
{
match
*
self
{
Integer
(
_
)
|
Err
(
_
)
=>
"an"
,
impl
LitKind
{
crate
fn
article
(
self
)
->
&
'static
str
{
match
self
{
Integer
|
Err
=>
"an"
,
_
=>
"a"
,
}
}
crate
fn
descr
(
&
self
)
->
&
'static
str
{
match
*
self
{
Bool
(
_
)
=>
panic!
(
"literal token contains `Lit::Bool`"
),
Byte
(
_
)
=>
"byte literal"
,
Char
(
_
)
=>
"char literal"
,
Err
(
_
)
=>
"invalid
literal"
,
Integer
(
_
)
=>
"integer
literal"
,
Float
(
_
)
=>
"float
literal"
,
Str_
(
_
)
|
StrRaw
(
..
)
=>
"
string literal"
,
ByteStr
(
_
)
|
ByteStrRaw
(
..
)
=>
"byte string literal"
crate
fn
descr
(
self
)
->
&
'static
str
{
match
self
{
Bool
=>
panic!
(
"literal token contains `Lit::Bool`"
),
Byte
=>
"byte literal"
,
Char
=>
"char literal"
,
Integer
=>
"integer
literal"
,
Float
=>
"float
literal"
,
Str
|
StrRaw
(
..
)
=>
"string
literal"
,
ByteStr
|
ByteStrRaw
(
..
)
=>
"byte
string literal"
,
Err
=>
"invalid literal"
,
}
}
crate
fn
may_have_suffix
(
&
self
)
->
bool
{
match
*
self
{
Integer
(
..
)
|
Float
(
..
)
|
Err
(
..
)
=>
true
,
crate
fn
may_have_suffix
(
self
)
->
bool
{
match
self
{
Integer
|
Float
|
Err
=>
true
,
_
=>
false
,
}
}
}
impl
Lit
{
pub
fn
new
(
kind
:
LitKind
,
symbol
:
Symbol
,
suffix
:
Option
<
Symbol
>
)
->
Lit
{
Lit
{
kind
,
symbol
,
suffix
}
}
}
pub
(
crate
)
fn
ident_can_begin_expr
(
ident
:
ast
::
Ident
,
is_raw
:
bool
)
->
bool
{
let
ident_token
:
Token
=
Ident
(
ident
,
is_raw
);
...
...
@@ -201,7 +204,7 @@ pub enum Token {
CloseDelim
(
DelimToken
),
/* Literals */
Literal
(
Lit
,
Option
<
ast
::
Name
>
),
Literal
(
Lit
),
/* Name components */
Ident
(
ast
::
Ident
,
/* is_raw */
bool
),
...
...
@@ -318,6 +321,10 @@ pub fn can_begin_const_arg(&self) -> bool {
self
==
&
Question
||
self
==
&
OpenDelim
(
Paren
)
}
pub
fn
lit
(
kind
:
LitKind
,
symbol
:
Symbol
,
suffix
:
Option
<
Symbol
>
)
->
Token
{
Literal
(
Lit
::
new
(
kind
,
symbol
,
suffix
))
}
/// Returns `true` if the token is any literal
crate
fn
is_lit
(
&
self
)
->
bool
{
match
*
self
{
...
...
@@ -326,9 +333,9 @@ pub fn can_begin_const_arg(&self) -> bool {
}
}
crate
fn
expect_lit
(
&
self
)
->
(
Lit
,
Option
<
Symbol
>
)
{
crate
fn
expect_lit
(
&
self
)
->
Lit
{
match
*
self
{
Literal
(
lit
,
suf
)
=>
(
lit
,
suf
)
,
Literal
(
lit
)
=>
lit
,
_
=>
panic!
(
"`expect_lit` called on non-literal"
),
}
}
...
...
@@ -579,13 +586,13 @@ pub fn is_reserved_ident(&self) -> bool {
(
&
DocComment
(
a
),
&
DocComment
(
b
))
|
(
&
Shebang
(
a
),
&
Shebang
(
b
))
=>
a
==
b
,
(
&
Literal
(
a
),
&
Literal
(
b
))
=>
a
==
b
,
(
&
Lifetime
(
a
),
&
Lifetime
(
b
))
=>
a
.name
==
b
.name
,
(
&
Ident
(
a
,
b
),
&
Ident
(
c
,
d
))
=>
b
==
d
&&
(
a
.name
==
c
.name
||
a
.name
==
kw
::
DollarCrate
||
c
.name
==
kw
::
DollarCrate
),
(
&
Literal
(
a
,
b
),
&
Literal
(
c
,
d
))
=>
b
==
d
&&
a
==
c
,
(
&
Interpolated
(
_
),
&
Interpolated
(
_
))
=>
false
,
_
=>
panic!
(
"forgot to add a token?"
),
...
...
src/libsyntax/print/pprust.rs
浏览文件 @
ca2a50fa
...
...
@@ -163,22 +163,22 @@ fn binop_to_string(op: BinOpToken) -> &'static str {
}
}
pub
fn
literal_to_string
(
lit
:
token
::
Lit
,
suffix
:
Option
<
ast
::
Name
>
)
->
String
{
let
mut
out
=
match
lit
{
token
::
Byte
(
b
)
=>
format!
(
"b'{}'"
,
b
),
token
::
Char
(
c
)
=>
format!
(
"'{}'"
,
c
),
token
::
Err
(
c
)
=>
format!
(
"'{}'"
,
c
),
token
::
Bool
(
c
)
|
token
::
Float
(
c
)
|
token
::
Integer
(
c
)
=>
c
.to_string
(
),
token
::
Str
_
(
s
)
=>
format!
(
"
\"
{}
\"
"
,
s
)
,
token
::
StrRaw
(
s
,
n
)
=>
format!
(
"r{delim}
\"
{string}
\"
{delim}"
,
delim
=
"#"
.repeat
(
n
as
usize
),
string
=
s
),
token
::
ByteStr
(
v
)
=>
format!
(
"b
\"
{}
\"
"
,
v
)
,
token
::
ByteStrRaw
(
s
,
n
)
=>
format!
(
"br{delim}
\"
{string}
\"
{delim}"
,
delim
=
"#"
.repeat
(
n
as
usize
),
string
=
s
),
pub
fn
literal_to_string
(
token
::
Lit
{
kind
,
symbol
,
suffix
}:
token
::
Lit
)
->
String
{
let
mut
out
=
match
kind
{
token
::
Byte
=>
format!
(
"b'{}'"
,
symbol
),
token
::
Char
=>
format!
(
"'{}'"
,
symbol
),
token
::
Bool
|
token
::
Float
|
token
::
Integer
=>
symbol
.to_string
(),
token
::
Str
=>
format!
(
"
\"
{}
\"
"
,
symbol
),
token
::
Str
Raw
(
n
)
=>
format!
(
"r{delim}
\"
{string}
\"
{delim}"
,
delim
=
"#"
.repeat
(
n
as
usize
)
,
string
=
symbol
),
token
::
ByteStr
=>
format!
(
"b
\"
{}
\"
"
,
symbol
),
token
::
ByteStr
Raw
(
n
)
=>
format!
(
"br{delim}
\"
{string}
\"
{delim}"
,
delim
=
"#"
.repeat
(
n
as
usize
)
,
string
=
symbol
),
token
::
Err
=>
format!
(
"'{}'"
,
symbol
),
};
if
let
Some
(
suffix
)
=
suffix
{
...
...
@@ -231,7 +231,7 @@ pub fn token_to_string(tok: &Token) -> String {
token
::
SingleQuote
=>
"'"
.to_string
(),
/* Literals */
token
::
Literal
(
lit
,
suf
)
=>
literal_to_string
(
lit
,
suf
),
token
::
Literal
(
lit
)
=>
literal_to_string
(
lit
),
/* Name components */
token
::
Ident
(
s
,
false
)
=>
s
.to_string
(),
...
...
@@ -571,7 +571,7 @@ fn next_comment(&mut self) -> Option<comments::Comment> {
fn
print_literal
(
&
mut
self
,
lit
:
&
ast
::
Lit
)
->
io
::
Result
<
()
>
{
self
.maybe_print_comment
(
lit
.span
.lo
())
?
;
self
.writer
()
.word
(
literal_to_string
(
lit
.token
,
lit
.suffix
))
self
.writer
()
.word
(
literal_to_string
(
lit
.token
))
}
fn
print_string
(
&
mut
self
,
st
:
&
str
,
...
...
src/libsyntax_ext/assert.rs
浏览文件 @
ca2a50fa
...
...
@@ -4,7 +4,7 @@
use
syntax
::
source_map
::
Spanned
;
use
syntax
::
ext
::
base
::
*
;
use
syntax
::
ext
::
build
::
AstBuilder
;
use
syntax
::
parse
::
token
;
use
syntax
::
parse
::
token
::{
self
,
Token
}
;
use
syntax
::
parse
::
parser
::
Parser
;
use
syntax
::
print
::
pprust
;
use
syntax
::
ptr
::
P
;
...
...
@@ -31,13 +31,10 @@ pub fn expand_assert<'cx>(
tts
:
custom_message
.unwrap_or_else
(||
{
TokenStream
::
from
(
TokenTree
::
Token
(
DUMMY_SP
,
token
::
Literal
(
token
::
Lit
::
Str_
(
Name
::
intern
(
&
format!
(
"assertion failed: {}"
,
pprust
::
expr_to_string
(
&
cond_expr
)
.escape_debug
()
))),
None
,
),
Token
::
lit
(
token
::
Str
,
Symbol
::
intern
(
&
format!
(
"assertion failed: {}"
,
pprust
::
expr_to_string
(
&
cond_expr
)
.escape_debug
()
)),
None
),
))
})
.into
(),
delim
:
MacDelimiter
::
Parenthesis
,
...
...
@@ -106,7 +103,7 @@ fn parse_assert<'a>(
//
// Parse this as an actual message, and suggest inserting a comma. Eventually, this should be
// turned into an error.
let
custom_message
=
if
let
token
::
Literal
(
token
::
Lit
::
Str_
(
_
),
_
)
=
parser
.token
{
let
custom_message
=
if
let
token
::
Literal
(
token
::
Lit
{
kind
:
token
::
Str
,
..
}
)
=
parser
.token
{
let
mut
err
=
cx
.struct_span_warn
(
parser
.span
,
"unexpected string literal"
);
let
comma_span
=
cx
.source_map
()
.next_point
(
parser
.prev_span
);
err
.span_suggestion_short
(
...
...
src/libsyntax_ext/proc_macro_server.rs
浏览文件 @
ca2a50fa
...
...
@@ -150,7 +150,7 @@ fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mu
stack
.push
(
tt!
(
Ident
::
new
(
ident
.name
,
false
)));
tt!
(
Punct
::
new
(
'\''
,
true
))
}
Literal
(
lit
,
suffix
)
=>
tt!
(
Literal
{
lit
,
suffix
}),
Literal
(
lit
)
=>
tt!
(
Literal
{
lit
}),
DocComment
(
c
)
=>
{
let
style
=
comments
::
doc_comment_style
(
&
c
.as_str
());
let
stripped
=
comments
::
strip_doc_comment_decoration
(
&
c
.as_str
());
...
...
@@ -161,7 +161,7 @@ fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mu
let
stream
=
vec!
[
Ident
(
ast
::
Ident
::
new
(
Symbol
::
intern
(
"doc"
),
span
),
false
),
Eq
,
Literal
(
Lit
::
Str_
(
Symbol
::
intern
(
&
escaped
)
),
None
),
Token
::
lit
(
token
::
Str
,
Symbol
::
intern
(
&
escaped
),
None
),
]
.into_iter
()
.map
(|
token
|
tokenstream
::
TokenTree
::
Token
(
span
,
token
))
...
...
@@ -215,31 +215,29 @@ fn to_internal(self) -> TokenStream {
return
tokenstream
::
TokenTree
::
Token
(
span
,
token
)
.into
();
}
TokenTree
::
Literal
(
self
::
Literal
{
lit
:
Lit
::
Integer
(
ref
a
),
suffix
,
lit
:
token
::
Lit
{
kind
:
token
::
Integer
,
symbol
,
suffix
},
span
,
})
if
a
.as_str
()
.starts_with
(
"-"
)
=>
{
})
if
symbol
.as_str
()
.starts_with
(
"-"
)
=>
{
let
minus
=
BinOp
(
BinOpToken
::
Minus
);
let
integer
=
Symbol
::
intern
(
&
a
.as_str
()[
1
..
]);
let
integer
=
Literal
(
Lit
::
Integer
(
integer
)
,
suffix
);
let
symbol
=
Symbol
::
intern
(
&
symbol
.as_str
()[
1
..
]);
let
integer
=
Token
::
lit
(
token
::
Integer
,
symbol
,
suffix
);
let
a
=
tokenstream
::
TokenTree
::
Token
(
span
,
minus
);
let
b
=
tokenstream
::
TokenTree
::
Token
(
span
,
integer
);
return
vec!
[
a
,
b
]
.into_iter
()
.collect
();
}
TokenTree
::
Literal
(
self
::
Literal
{
lit
:
Lit
::
Float
(
ref
a
),
suffix
,
lit
:
token
::
Lit
{
kind
:
token
::
Float
,
symbol
,
suffix
},
span
,
})
if
a
.as_str
()
.starts_with
(
"-"
)
=>
{
})
if
symbol
.as_str
()
.starts_with
(
"-"
)
=>
{
let
minus
=
BinOp
(
BinOpToken
::
Minus
);
let
float
=
Symbol
::
intern
(
&
a
.as_str
()[
1
..
]);
let
float
=
Literal
(
Lit
::
Float
(
float
)
,
suffix
);
let
symbol
=
Symbol
::
intern
(
&
symbol
.as_str
()[
1
..
]);
let
float
=
Token
::
lit
(
token
::
Float
,
symbol
,
suffix
);
let
a
=
tokenstream
::
TokenTree
::
Token
(
span
,
minus
);
let
b
=
tokenstream
::
TokenTree
::
Token
(
span
,
float
);
return
vec!
[
a
,
b
]
.into_iter
()
.collect
();
}
TokenTree
::
Literal
(
self
::
Literal
{
lit
,
s
uffix
,
s
pan
})
=>
{
return
tokenstream
::
TokenTree
::
Token
(
span
,
Literal
(
lit
,
suffix
))
.into
()
TokenTree
::
Literal
(
self
::
Literal
{
lit
,
span
})
=>
{
return
tokenstream
::
TokenTree
::
Token
(
span
,
Literal
(
lit
))
.into
()
}
};
...
...
@@ -355,7 +353,6 @@ fn dollar_crate(span: Span) -> Ident {
#[derive(Clone,
Debug)]
pub
struct
Literal
{
lit
:
token
::
Lit
,
suffix
:
Option
<
Symbol
>
,
span
:
Span
,
}
...
...
@@ -381,6 +378,13 @@ pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
call_site
:
to_span
(
Transparency
::
Transparent
),
}
}
pub
fn
lit
(
&
mut
self
,
kind
:
token
::
LitKind
,
symbol
:
Symbol
,
suffix
:
Option
<
Symbol
>
)
->
Literal
{
Literal
{
lit
:
token
::
Lit
::
new
(
kind
,
symbol
,
suffix
),
span
:
server
::
Span
::
call_site
(
self
),
}
}
}
impl
server
::
Types
for
Rustc
<
'_
>
{
...
...
@@ -536,59 +540,31 @@ fn debug(&mut self, literal: &Self::Literal) -> String {
format!
(
"{:?}"
,
literal
)
}
fn
integer
(
&
mut
self
,
n
:
&
str
)
->
Self
::
Literal
{
Literal
{
lit
:
token
::
Lit
::
Integer
(
Symbol
::
intern
(
n
)),
suffix
:
None
,
span
:
server
::
Span
::
call_site
(
self
),
}
self
.lit
(
token
::
Integer
,
Symbol
::
intern
(
n
),
None
)
}
fn
typed_integer
(
&
mut
self
,
n
:
&
str
,
kind
:
&
str
)
->
Self
::
Literal
{
Literal
{
lit
:
token
::
Lit
::
Integer
(
Symbol
::
intern
(
n
)),
suffix
:
Some
(
Symbol
::
intern
(
kind
)),
span
:
server
::
Span
::
call_site
(
self
),
}
self
.lit
(
token
::
Integer
,
Symbol
::
intern
(
n
),
Some
(
Symbol
::
intern
(
kind
)))
}
fn
float
(
&
mut
self
,
n
:
&
str
)
->
Self
::
Literal
{
Literal
{
lit
:
token
::
Lit
::
Float
(
Symbol
::
intern
(
n
)),
suffix
:
None
,
span
:
server
::
Span
::
call_site
(
self
),
}
self
.lit
(
token
::
Float
,
Symbol
::
intern
(
n
),
None
)
}
fn
f32
(
&
mut
self
,
n
:
&
str
)
->
Self
::
Literal
{
Literal
{
lit
:
token
::
Lit
::
Float
(
Symbol
::
intern
(
n
)),
suffix
:
Some
(
Symbol
::
intern
(
"f32"
)),
span
:
server
::
Span
::
call_site
(
self
),
}
self
.lit
(
token
::
Float
,
Symbol
::
intern
(
n
),
Some
(
Symbol
::
intern
(
"f32"
)))
}
fn
f64
(
&
mut
self
,
n
:
&
str
)
->
Self
::
Literal
{
Literal
{
lit
:
token
::
Lit
::
Float
(
Symbol
::
intern
(
n
)),
suffix
:
Some
(
Symbol
::
intern
(
"f64"
)),
span
:
server
::
Span
::
call_site
(
self
),
}
self
.lit
(
token
::
Float
,
Symbol
::
intern
(
n
),
Some
(
Symbol
::
intern
(
"f64"
)))
}
fn
string
(
&
mut
self
,
string
:
&
str
)
->
Self
::
Literal
{
let
mut
escaped
=
String
::
new
();
for
ch
in
string
.chars
()
{
escaped
.extend
(
ch
.escape_debug
());
}
Literal
{
lit
:
token
::
Lit
::
Str_
(
Symbol
::
intern
(
&
escaped
)),
suffix
:
None
,
span
:
server
::
Span
::
call_site
(
self
),
}
self
.lit
(
token
::
Str
,
Symbol
::
intern
(
&
escaped
),
None
)
}
fn
character
(
&
mut
self
,
ch
:
char
)
->
Self
::
Literal
{
let
mut
escaped
=
String
::
new
();
escaped
.extend
(
ch
.escape_unicode
());
Literal
{
lit
:
token
::
Lit
::
Char
(
Symbol
::
intern
(
&
escaped
)),
suffix
:
None
,
span
:
server
::
Span
::
call_site
(
self
),
}
self
.lit
(
token
::
Char
,
Symbol
::
intern
(
&
escaped
),
None
)
}
fn
byte_string
(
&
mut
self
,
bytes
:
&
[
u8
])
->
Self
::
Literal
{
let
string
=
bytes
...
...
@@ -597,11 +573,7 @@ fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
.flat_map
(
ascii
::
escape_default
)
.map
(
Into
::
<
char
>
::
into
)
.collect
::
<
String
>
();
Literal
{
lit
:
token
::
Lit
::
ByteStr
(
Symbol
::
intern
(
&
string
)),
suffix
:
None
,
span
:
server
::
Span
::
call_site
(
self
),
}
self
.lit
(
token
::
ByteStr
,
Symbol
::
intern
(
&
string
),
None
)
}
fn
span
(
&
mut
self
,
literal
:
&
Self
::
Literal
)
->
Self
::
Span
{
literal
.span
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录