Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
int
Rust
提交
13bc8afa
R
Rust
项目概览
int
/
Rust
接近 1 年 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
R
Rust
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
13bc8afa
编写于
4月 21, 2015
作者:
P
Piotr Czarnecki
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Model lexer: Fix remaining issues
上级
e5e343ae
变更
57
隐藏空白更改
内联
并排
Showing
57 changed file
with
102 addition
and
179 deletion
+102
-179
src/grammar/README.md
src/grammar/README.md
+1
-1
src/grammar/RustLexer.g4
src/grammar/RustLexer.g4
+47
-65
src/grammar/check.sh
src/grammar/check.sh
+4
-4
src/grammar/verify.rs
src/grammar/verify.rs
+50
-34
src/libcollections/fmt.rs
src/libcollections/fmt.rs
+0
-2
src/libcollections/str.rs
src/libcollections/str.rs
+0
-2
src/libcollections/string.rs
src/libcollections/string.rs
+0
-2
src/libcore/hash/sip.rs
src/libcore/hash/sip.rs
+0
-2
src/libcore/num/mod.rs
src/libcore/num/mod.rs
+0
-2
src/libcore/str/mod.rs
src/libcore/str/mod.rs
+0
-2
src/libcoretest/char.rs
src/libcoretest/char.rs
+0
-2
src/libgetopts/lib.rs
src/libgetopts/lib.rs
+0
-2
src/librand/distributions/gamma.rs
src/librand/distributions/gamma.rs
+0
-2
src/librustc_unicode/u_str.rs
src/librustc_unicode/u_str.rs
+0
-2
src/libserialize/hex.rs
src/libserialize/hex.rs
+0
-2
src/libstd/ascii.rs
src/libstd/ascii.rs
+0
-2
src/libstd/collections/hash/map.rs
src/libstd/collections/hash/map.rs
+0
-2
src/libstd/collections/hash/set.rs
src/libstd/collections/hash/set.rs
+0
-2
src/libstd/collections/hash/table.rs
src/libstd/collections/hash/table.rs
+0
-2
src/libstd/io/buffered.rs
src/libstd/io/buffered.rs
+0
-2
src/libstd/num/strconv.rs
src/libstd/num/strconv.rs
+0
-2
src/libstd/rt/util.rs
src/libstd/rt/util.rs
+0
-2
src/libsyntax/codemap.rs
src/libsyntax/codemap.rs
+0
-2
src/libsyntax/ext/tt/macro_parser.rs
src/libsyntax/ext/tt/macro_parser.rs
+0
-2
src/test/bench/core-std.rs
src/test/bench/core-std.rs
+0
-1
src/test/bench/msgsend-ring-mutex-arcs.rs
src/test/bench/msgsend-ring-mutex-arcs.rs
+0
-1
src/test/bench/noise.rs
src/test/bench/noise.rs
+0
-1
src/test/compile-fail/utf8_idents.rs
src/test/compile-fail/utf8_idents.rs
+0
-1
src/test/pretty/block-comment-wchar.pp
src/test/pretty/block-comment-wchar.pp
+0
-1
src/test/pretty/block-comment-wchar.rs
src/test/pretty/block-comment-wchar.rs
+0
-1
src/test/run-pass/byte-literals.rs
src/test/run-pass/byte-literals.rs
+0
-1
src/test/run-pass/default-method-supertrait-vtable.rs
src/test/run-pass/default-method-supertrait-vtable.rs
+0
-1
src/test/run-pass/ifmt.rs
src/test/run-pass/ifmt.rs
+0
-1
src/test/run-pass/issue-12582.rs
src/test/run-pass/issue-12582.rs
+0
-1
src/test/run-pass/issue-13027.rs
src/test/run-pass/issue-13027.rs
+0
-1
src/test/run-pass/issue-2718.rs
src/test/run-pass/issue-2718.rs
+0
-1
src/test/run-pass/issue-3683.rs
src/test/run-pass/issue-3683.rs
+0
-1
src/test/run-pass/issue-4759-1.rs
src/test/run-pass/issue-4759-1.rs
+0
-2
src/test/run-pass/issue-5280.rs
src/test/run-pass/issue-5280.rs
+0
-1
src/test/run-pass/issue-5321-immediates-with-bare-self.rs
src/test/run-pass/issue-5321-immediates-with-bare-self.rs
+0
-1
src/test/run-pass/lexer-crlf-line-endings-string-literal-doc-comment.rs
...ass/lexer-crlf-line-endings-string-literal-doc-comment.rs
+0
-1
src/test/run-pass/lint-non-camel-case-types-non-uppercase-statics-unicode.rs
...int-non-camel-case-types-non-uppercase-statics-unicode.rs
+0
-1
src/test/run-pass/match-range.rs
src/test/run-pass/match-range.rs
+0
-1
src/test/run-pass/multibyte.rs
src/test/run-pass/multibyte.rs
+0
-1
src/test/run-pass/raw-str.rs
src/test/run-pass/raw-str.rs
+0
-0
src/test/run-pass/shebang.rs
src/test/run-pass/shebang.rs
+0
-1
src/test/run-pass/struct-return.rs
src/test/run-pass/struct-return.rs
+0
-1
src/test/run-pass/trait-to-str.rs
src/test/run-pass/trait-to-str.rs
+0
-1
src/test/run-pass/trait-with-bounds-default.rs
src/test/run-pass/trait-with-bounds-default.rs
+0
-1
src/test/run-pass/traits-default-method-self.rs
src/test/run-pass/traits-default-method-self.rs
+0
-1
src/test/run-pass/traits-default-method-trivial.rs
src/test/run-pass/traits-default-method-trivial.rs
+0
-1
src/test/run-pass/unsized.rs
src/test/run-pass/unsized.rs
+0
-2
src/test/run-pass/unsized2.rs
src/test/run-pass/unsized2.rs
+0
-2
src/test/run-pass/utf8-bom.rs
src/test/run-pass/utf8-bom.rs
+0
-1
src/test/run-pass/utf8.rs
src/test/run-pass/utf8.rs
+0
-1
src/test/run-pass/utf8_chars.rs
src/test/run-pass/utf8_chars.rs
+0
-1
src/test/run-pass/utf8_idents.rs
src/test/run-pass/utf8_idents.rs
+0
-1
未找到文件。
src/grammar/README.md
浏览文件 @
13bc8afa
...
...
@@ -12,7 +12,7 @@ javac *.java
rustc -O verify.rs
for file in ../*/**.rs; do
echo $file;
grun RustLexer tokens -tokens <
$file | ./verify $file
RustLexer.tokens || break
grun RustLexer tokens -tokens <
"$file" | ./verify "$file"
RustLexer.tokens || break
done
```
...
...
src/grammar/RustLexer.g4
浏览文件 @
13bc8afa
lexer grammar RustLexer;
@lexer::members {
public boolean is_at(int pos) {
return _input.index() == pos;
}
}
tokens {
EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, PLUT,
MINUS, STAR, SLASH, PERCENT, CARET, AND, OR, SHL, SHR, BINOP,
...
...
@@ -8,7 +15,7 @@ tokens {
LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR,
LIT_INTEGER, LIT_FLOAT, LIT_STR, LIT_STR_RAW, LIT_BINARY,
LIT_BINARY_RAW, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT,
COMMENT
COMMENT
, SHEBANG
}
import xidstart , xidcontinue;
...
...
@@ -86,85 +93,54 @@ fragment CHAR_ESCAPE
| [xX] HEXIT HEXIT
| 'u' HEXIT HEXIT HEXIT HEXIT
| 'U' HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT
| 'u{' HEXIT '}'
| 'u{' HEXIT HEXIT '}'
| 'u{' HEXIT HEXIT HEXIT '}'
| 'u{' HEXIT HEXIT HEXIT HEXIT '}'
| 'u{' HEXIT HEXIT HEXIT HEXIT HEXIT '}'
| 'u{' HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT '}'
;
fragment SUFFIX
: IDENT
;
fragment INTEGER_SUFFIX
: { _input.LA(1) != 'e' && _input.LA(1) != 'E' }? SUFFIX
;
LIT_CHAR
: '\'' ( '\\' CHAR_ESCAPE | ~[\\'\n\t\r] | '\ud800' .. '\udbff' '\udc00' .. '\udfff' ) '\'' SUFFIX?
: '\'' ( '\\' CHAR_ESCAPE
| ~[\\'\n\t\r]
| '\ud800' .. '\udbff' '\udc00' .. '\udfff'
)
'\'' SUFFIX?
;
LIT_BYTE
: 'b\'' ( '\\' ( [xX] HEXIT HEXIT | [nrt\\'"0] ) | ~[\\'\n\t\r] ) '\'' SUFFIX?
: 'b\'' ( '\\' ( [xX] HEXIT HEXIT
| [nrt\\'"0] )
| ~[\\'\n\t\r] '\udc00'..'\udfff'?
)
'\'' SUFFIX?
;
LIT_INTEGER
: [0-9][0-9_]* SUFFIX?
| '0b' [01][01_]* SUFFIX?
| '0o' [0-7][0-7_]* SUFFIX?
| '0x' [0-9a-fA-F][0-9a-fA-F_]* SUFFIX?
: [0-9][0-9_]* INTEGER_SUFFIX?
| '0b' [01_]+ INTEGER_SUFFIX?
| '0o' [0-7_]+ INTEGER_SUFFIX?
| '0x' [0-9a-fA-F_]+ INTEGER_SUFFIX?
;
LIT_FLOAT
: [0-9][0-9_]* ('.' {
/* dot followed by another dot is a range, no float */
/* dot followed by another dot is a range, no
t a
float */
_input.LA(1) != '.' &&
/* dot followed by an identifier is an integer with a function call, no float */
/* dot followed by an identifier is an integer with a function call, no
t a
float */
_input.LA(1) != '_' &&
_input.LA(1) != 'a' &&
_input.LA(1) != 'b' &&
_input.LA(1) != 'c' &&
_input.LA(1) != 'd' &&
_input.LA(1) != 'e' &&
_input.LA(1) != 'f' &&
_input.LA(1) != 'g' &&
_input.LA(1) != 'h' &&
_input.LA(1) != 'i' &&
_input.LA(1) != 'j' &&
_input.LA(1) != 'k' &&
_input.LA(1) != 'l' &&
_input.LA(1) != 'm' &&
_input.LA(1) != 'n' &&
_input.LA(1) != 'o' &&
_input.LA(1) != 'p' &&
_input.LA(1) != 'q' &&
_input.LA(1) != 'r' &&
_input.LA(1) != 's' &&
_input.LA(1) != 't' &&
_input.LA(1) != 'u' &&
_input.LA(1) != 'v' &&
_input.LA(1) != 'w' &&
_input.LA(1) != 'x' &&
_input.LA(1) != 'y' &&
_input.LA(1) != 'z' &&
_input.LA(1) != 'A' &&
_input.LA(1) != 'B' &&
_input.LA(1) != 'C' &&
_input.LA(1) != 'D' &&
_input.LA(1) != 'E' &&
_input.LA(1) != 'F' &&
_input.LA(1) != 'G' &&
_input.LA(1) != 'H' &&
_input.LA(1) != 'I' &&
_input.LA(1) != 'J' &&
_input.LA(1) != 'K' &&
_input.LA(1) != 'L' &&
_input.LA(1) != 'M' &&
_input.LA(1) != 'N' &&
_input.LA(1) != 'O' &&
_input.LA(1) != 'P' &&
_input.LA(1) != 'Q' &&
_input.LA(1) != 'R' &&
_input.LA(1) != 'S' &&
_input.LA(1) != 'T' &&
_input.LA(1) != 'U' &&
_input.LA(1) != 'V' &&
_input.LA(1) != 'W' &&
_input.LA(1) != 'X' &&
_input.LA(1) != 'Y' &&
_input.LA(1) != 'Z'
!(_input.LA(1) >= 'a' && _input.LA(1) <= 'z') &&
!(_input.LA(1) >= 'A' && _input.LA(1) <= 'Z')
}? | ('.' [0-9][0-9_]*)? ([eE] [-+]? [0-9][0-9_]*)? SUFFIX?)
;
...
...
@@ -172,8 +148,8 @@ LIT_STR
: '"' ('\\\n' | '\\\r\n' | '\\' CHAR_ESCAPE | .)*? '"' SUFFIX?
;
LIT_BINARY : 'b' LIT_STR
SUFFIX?
;
LIT_BINARY_RAW : '
rb' LIT_STR_RAW SUFFIX?
;
LIT_BINARY : 'b' LIT_STR ;
LIT_BINARY_RAW : '
b' LIT_STR_RAW
;
/* this is a bit messy */
...
...
@@ -201,13 +177,19 @@ LIFETIME : '\'' IDENT ;
WHITESPACE : [ \r\n\t]+ ;
UNDOC_COMMENT : '////' ~[\
r\
n]* -> type(COMMENT) ;
UNDOC_COMMENT : '////' ~[\n]* -> type(COMMENT) ;
YESDOC_COMMENT : '///' ~[\r\n]* -> type(DOC_COMMENT) ;
OUTER_DOC_COMMENT : '//!' ~[\r\n]* -> type(DOC_COMMENT) ;
LINE_COMMENT : '//'
~[\r\n]*
-> type(COMMENT) ;
LINE_COMMENT : '//'
( ~[/\n] ~[\n]* )?
-> type(COMMENT) ;
DOC_BLOCK_COMMENT
: ('/**' ~[*] | '/*!') (DOC_BLOCK_COMMENT | .)*? '*/' -> type(DOC_COMMENT)
;
BLOCK_COMMENT : '/*' (BLOCK_COMMENT | .)*? '*/' -> type(COMMENT) ;
/* these appear at the beginning of a file */
SHEBANG : '#!' { is_at(2) && _input.LA(1) != '[' }? ~[\r\n]* -> type(SHEBANG) ;
UTF8_BOM : '\ufeff' { is_at(1) }? -> skip ;
src/grammar/check.sh
浏览文件 @
13bc8afa
...
...
@@ -18,13 +18,13 @@ failed=0
skipped
=
0
check
()
{
grep
--silent
"// ignore-lexer-test"
$1
;
grep
--silent
"// ignore-lexer-test"
"
$1
"
;
# if it's *not* found...
if
[
$?
-eq
1
]
;
then
cd
$2
# This `cd` is so java will pick up RustLexer.class. I couldn't
# figure out how to wrangle the CLASSPATH, just adding build/gramm
r didn't
#
seem to have an
ny effect.
# figure out how to wrangle the CLASSPATH, just adding build/gramm
ar
#
didn't seem to have a
ny effect.
if
$3
RustLexer tokens
-tokens
<
$1
|
$4
$1
$5
;
then
echo
"pass:
$1
"
passed
=
`
expr
$passed
+ 1
`
...
...
@@ -39,7 +39,7 @@ check() {
}
for
file
in
$(
find
$1
-iname
'*.rs'
!
-path
'*/test/compile-fail*'
)
;
do
check
$file
$2
$3
$4
$5
check
"
$file
"
$2
$3
$4
$5
done
printf
"
\n
test result: "
...
...
src/grammar/verify.rs
浏览文件 @
13bc8afa
...
...
@@ -8,9 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#![feature(plugin)]
#![allow(unstable)]
#![feature(plugin,
rustc_private,
str_char,
collections)]
extern
crate
syntax
;
extern
crate
rustc
;
...
...
@@ -19,7 +17,10 @@
extern
crate
log
;
use
std
::
collections
::
HashMap
;
use
std
::
io
::
File
;
use
std
::
env
;
use
std
::
fs
::
File
;
use
std
::
io
::{
BufRead
,
Read
};
use
std
::
path
::
Path
;
use
syntax
::
parse
;
use
syntax
::
parse
::
lexer
;
...
...
@@ -27,6 +28,7 @@
use
syntax
::
ast
;
use
syntax
::
ast
::
Name
;
use
syntax
::
codemap
;
use
syntax
::
codemap
::
Pos
;
use
syntax
::
parse
::
token
;
use
syntax
::
parse
::
lexer
::
TokenAndSpan
;
...
...
@@ -108,6 +110,7 @@ fn id() -> token::Token {
"LIT_BINARY"
=>
token
::
Literal
(
token
::
Binary
(
Name
(
0
)),
None
),
"LIT_BINARY_RAW"
=>
token
::
Literal
(
token
::
BinaryRaw
(
Name
(
0
),
0
),
None
),
"QUESTION"
=>
token
::
Question
,
"SHEBANG"
=>
token
::
Shebang
(
Name
(
0
)),
_
=>
continue
,
};
...
...
@@ -166,24 +169,26 @@ fn count(lit: &str) -> usize {
lit
.chars
()
.take_while
(|
c
|
*
c
==
'#'
)
.count
()
}
fn
parse_antlr_token
(
s
:
&
str
,
tokens
:
&
HashMap
<
String
,
token
::
Token
>
,
surrogate_pairs_pos
:
&
[
usize
])
fn
parse_antlr_token
(
s
:
&
str
,
tokens
:
&
HashMap
<
String
,
token
::
Token
>
,
surrogate_pairs_pos
:
&
[
usize
],
has_bom
:
bool
)
->
TokenAndSpan
{
// old regex:
// \[@(?P<seq>\d+),(?P<start>\d+):(?P<end>\d+)='(?P<content>.+?)',<(?P<toknum>-?\d+)>,\d+:\d+]
let
start
=
s
.find_str
(
"[@"
)
.unwrap
();
let
comma
=
start
+
s
[
start
..
]
.find_str
(
","
)
.unwrap
();
let
colon
=
comma
+
s
[
comma
..
]
.find_str
(
":"
)
.unwrap
();
let
content_start
=
colon
+
s
[
colon
..
]
.find_str
(
"='"
)
.unwrap
();
let
content_end
=
content_start
+
s
[
content_start
..
]
.find_str
(
"',<"
)
.unwrap
();
let
toknum_end
=
content_end
+
s
[
content_end
..
]
.find_str
(
">,"
)
.unwrap
();
let
start
=
s
.find
(
"[@"
)
.unwrap
();
let
comma
=
start
+
s
[
start
..
]
.find
(
","
)
.unwrap
();
let
colon
=
comma
+
s
[
comma
..
]
.find
(
":"
)
.unwrap
();
let
content_start
=
colon
+
s
[
colon
..
]
.find
(
"='"
)
.unwrap
();
// Use rfind instead of find, because we don't want to stop at the content
let
content_end
=
content_start
+
s
[
content_start
..
]
.rfind
(
"',<"
)
.unwrap
();
let
toknum_end
=
content_end
+
s
[
content_end
..
]
.find
(
">,"
)
.unwrap
();
let
start
=
&
s
[
comma
+
1
..
colon
];
let
end
=
&
s
[
colon
+
1
..
content_start
];
let
content
=
&
s
[
content_start
+
2
..
content_end
];
let
toknum
=
&
s
[
content_end
+
3
..
toknum_end
];
let
proto_tok
=
tokens
.get
(
toknum
)
.expect
(
format!
(
"didn't find token {:?} in the map"
,
toknum
)
);
let
not_found
=
format!
(
"didn't find token {:?} in the map"
,
toknum
);
let
proto_tok
=
tokens
.get
(
toknum
)
.expect
(
&
not_found
[
..
]
);
let
nm
=
parse
::
token
::
intern
(
content
);
...
...
@@ -209,24 +214,25 @@ fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_
ref
t
=>
t
.clone
()
};
let
offset
=
if
real_tok
==
token
::
Eof
{
let
start_offset
=
if
real_tok
==
token
::
Eof
{
1
}
else
{
0
};
let
mut
lo
=
start
.parse
::
<
u32
>
()
.unwrap
()
-
offset
;
let
mut
hi
=
end
.parse
::
<
u32
>
()
.unwrap
()
+
1
;
let
offset
=
if
has_bom
{
1
}
else
{
0
};
let
mut
lo
=
start
.parse
::
<
u32
>
()
.unwrap
()
-
start_offset
-
offset
;
let
mut
hi
=
end
.parse
::
<
u32
>
()
.unwrap
()
+
1
-
offset
;
// Adjust the span: For each surrogate pair already encountered, subtract one position.
lo
-=
surrogate_pairs_pos
.binary_search
(
&
(
lo
as
usize
))
.unwrap_or_else
(|
x
|
x
)
as
u32
;
hi
-=
surrogate_pairs_pos
.binary_search
(
&
(
hi
as
usize
))
.unwrap_or_else
(|
x
|
x
)
as
u32
;
let
sp
=
syntax
::
codemap
::
Span
{
lo
:
syntax
::
codemap
::
BytePos
(
lo
),
hi
:
syntax
::
codemap
::
BytePos
(
hi
),
expn_id
:
syntax
::
codemap
::
NO_EXPANSION
let
sp
=
codemap
::
Span
{
lo
:
codemap
::
BytePos
(
lo
),
hi
:
codemap
::
BytePos
(
hi
),
expn_id
:
codemap
::
NO_EXPANSION
};
TokenAndSpan
{
...
...
@@ -245,10 +251,10 @@ fn tok_cmp(a: &token::Token, b: &token::Token) -> bool {
}
}
fn
span_cmp
(
antlr_sp
:
syntax
::
codemap
::
Span
,
rust_sp
:
syntax
::
codemap
::
Span
,
cm
:
&
syntax
::
codemap
::
CodeMap
)
->
bool
{
fn
span_cmp
(
antlr_sp
:
codemap
::
Span
,
rust_sp
:
codemap
::
Span
,
cm
:
&
codemap
::
CodeMap
)
->
bool
{
antlr_sp
.expn_id
==
rust_sp
.expn_id
&&
antlr_sp
.lo
.to_u
int
()
==
cm
.bytepos_to_file_charpos
(
rust_sp
.lo
)
.to_uint
()
&&
antlr_sp
.hi
.to_u
int
()
==
cm
.bytepos_to_file_charpos
(
rust_sp
.hi
)
.to_uint
()
antlr_sp
.lo
.to_u
size
()
==
cm
.bytepos_to_file_charpos
(
rust_sp
.lo
)
.to_usize
()
&&
antlr_sp
.hi
.to_u
size
()
==
cm
.bytepos_to_file_charpos
(
rust_sp
.hi
)
.to_usize
()
}
fn
main
()
{
...
...
@@ -257,10 +263,15 @@ fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
r
.next_token
()
}
let
args
=
std
::
os
::
args
();
let
mut
args
=
env
::
args
()
.skip
(
1
);
let
filename
=
args
.next
()
.unwrap
();
if
filename
.find
(
"parse-fail"
)
.is_some
()
{
return
;
}
// Rust's lexer
let
code
=
File
::
open
(
&
Path
::
new
(
args
[
1
]))
.unwrap
()
.read_to_string
()
.unwrap
();
let
mut
code
=
String
::
new
();
File
::
open
(
&
Path
::
new
(
&
filename
))
.unwrap
()
.read_to_string
(
&
mut
code
)
.unwrap
();
let
surrogate_pairs_pos
:
Vec
<
usize
>
=
code
.chars
()
.enumerate
()
.filter
(|
&
(
_
,
c
)|
c
as
usize
>
0xFFFF
)
...
...
@@ -269,6 +280,8 @@ fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
.map
(|(
x
,
n
)|
x
+
n
)
.collect
();
let
has_bom
=
code
.starts_with
(
"
\
u{feff}"
);
debug!
(
"Pairs: {:?}"
,
surrogate_pairs_pos
);
let
options
=
config
::
basic_options
();
...
...
@@ -281,15 +294,18 @@ fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
let
ref
cm
=
lexer
.span_diagnostic.cm
;
// ANTLR
let
mut
token_file
=
File
::
open
(
&
Path
::
new
(
args
[
2
]));
let
token_map
=
parse_token_list
(
token_file
.read_to_string
()
.unwrap
());
let
mut
token_file
=
File
::
open
(
&
Path
::
new
(
&
args
.next
()
.unwrap
()))
.unwrap
();
let
mut
token_list
=
String
::
new
();
token_file
.read_to_string
(
&
mut
token_list
)
.unwrap
();
let
token_map
=
parse_token_list
(
&
token_list
[
..
]);
let
mut
stdin
=
std
::
io
::
stdin
();
let
mut
lock
=
stdin
.lock
();
let
stdin
=
std
::
io
::
stdin
();
let
lock
=
stdin
.lock
();
let
lines
=
lock
.lines
();
let
mut
antlr_tokens
=
lines
.map
(|
l
|
parse_antlr_token
(
l
.unwrap
()
.trim
(),
&
token_map
,
&
surrogate_pairs_pos
[]));
let
antlr_tokens
=
lines
.map
(|
l
|
parse_antlr_token
(
l
.unwrap
()
.trim
(),
&
token_map
,
&
surrogate_pairs_pos
[
..
],
has_bom
));
for
antlr_tok
in
antlr_tokens
{
let
rustc_tok
=
next
(
&
mut
lexer
);
...
...
@@ -314,7 +330,7 @@ fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
}
_
=>
panic!
(
"{:?} is not {:?}"
,
antlr_tok
,
rustc_tok
)
},)
*
ref
c
=>
assert!
(
c
==
&
antlr_tok
.tok
,
"{:?} is not {:?}"
,
rustc_tok
,
antlr
_tok
)
ref
c
=>
assert!
(
c
==
&
antlr_tok
.tok
,
"{:?} is not {:?}"
,
antlr_tok
,
rustc
_tok
)
}
)
}
...
...
src/libcollections/fmt.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! Utilities for formatting and printing strings
//!
...
...
src/libcollections/str.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! Unicode string manipulation (the `str` type).
//!
...
...
src/libcollections/string.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! An owned, growable string that enforces that its contents are valid UTF-8.
...
...
src/libcore/hash/sip.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
//! An implementation of SipHash 2-4.
...
...
src/libcore/num/mod.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! Numeric traits and functions for the built-in numeric types.
...
...
src/libcore/str/mod.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! String manipulation
//!
...
...
src/libcoretest/char.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
#[test]
fn
test_is_lowercase
()
{
...
...
src/libgetopts/lib.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15677
//! Simple getopt alternative.
//!
...
...
src/librand/distributions/gamma.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! The Gamma and derived distributions.
...
...
src/librustc_unicode/u_str.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! Unicode-intensive string manipulations.
//!
...
...
src/libserialize/hex.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! Hex binary-to-text encoding
...
...
src/libstd/ascii.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! Operations on ASCII strings and characters
...
...
src/libstd/collections/hash/map.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
use
self
::
Entry
::
*
;
use
self
::
SearchResult
::
*
;
...
...
src/libstd/collections/hash/set.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
use
borrow
::
Borrow
;
use
clone
::
Clone
;
...
...
src/libstd/collections/hash/table.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
use
self
::
BucketState
::
*
;
...
...
src/libstd/io/buffered.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
//! Buffering wrappers for I/O traits
...
...
src/libstd/num/strconv.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
#![allow(missing_docs)]
#![allow(deprecated)]
...
...
src/libstd/rt/util.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15677
use
io
::
prelude
::
*
;
...
...
src/libsyntax/codemap.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! The CodeMap tracks all the source code used within a single crate, mapping
//! from integer byte positions to the original source code location. Each bit
...
...
src/libsyntax/ext/tt/macro_parser.rs
浏览文件 @
13bc8afa
...
...
@@ -7,8 +7,6 @@
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
//! This is an Earley-like parser, without support for in-grammar nonterminals,
//! only by calling out to the main rust parser for named nonterminals (which it
...
...
src/test/bench/core-std.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// ignore-lexer-test FIXME #15679
// Microbenchmarks for various functions in std and extra
#![feature(rand,
collections,
std_misc)]
...
...
src/test/bench/msgsend-ring-mutex-arcs.rs
浏览文件 @
13bc8afa
...
...
@@ -16,7 +16,6 @@
// This also serves as a pipes test, because Arcs are implemented with pipes.
// no-pretty-expanded FIXME #15189
// ignore-lexer-test FIXME #15679
#![feature(std_misc)]
...
...
src/test/bench/noise.rs
浏览文件 @
13bc8afa
...
...
@@ -10,7 +10,6 @@
// Multi-language Perlin noise benchmark.
// See https://github.com/nsf/pnoise for timings and alternative implementations.
// ignore-lexer-test FIXME #15679
#![feature(rand,
core)]
...
...
src/test/compile-fail/utf8_idents.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
fn
foo
<
'β
,
//~ ERROR non-ascii idents are not fully supported
...
...
src/test/pretty/block-comment-wchar.pp
浏览文件 @
13bc8afa
...
...
@@ -14,7 +14,6 @@
//
ignore
-
tidy
-
cr
//
ignore
-
tidy
-
tab
//
pp
-
exact
:
block
-
comment
-
wchar
.
pp
//
ignore
-
lexer
-
test
FIXME
#15679
fn
f
()
{
fn
nested
()
{
/*
...
...
src/test/pretty/block-comment-wchar.rs
浏览文件 @
13bc8afa
...
...
@@ -14,7 +14,6 @@
// ignore-tidy-cr
// ignore-tidy-tab
// pp-exact:block-comment-wchar.pp
// ignore-lexer-test FIXME #15679
fn
f
()
{
fn
nested
()
{
/*
...
...
src/test/run-pass/byte-literals.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
static
FOO
:
u8
=
b
'\xF0'
;
...
...
src/test/run-pass/default-method-supertrait-vtable.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
// Tests that we can call a function bounded over a supertrait from
...
...
src/test/run-pass/ifmt.rs
浏览文件 @
13bc8afa
...
...
@@ -9,7 +9,6 @@
// except according to those terms.
// no-pretty-expanded unnecessary unsafe block generated
// ignore-lexer-test FIXME #15679
#![deny(warnings)]
#![allow(unused_must_use)]
...
...
src/test/run-pass/issue-12582.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
pub
fn
main
()
{
let
x
=
1
;
...
...
src/test/run-pass/issue-13027.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
// Tests that match expression handles overlapped literal and range
// properly in the presence of guard function.
...
...
src/test/run-pass/issue-2718.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
#![feature(unsafe_destructor,
std_misc)]
...
...
src/test/run-pass/issue-3683.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
trait
Foo
{
...
...
src/test/run-pass/issue-4759-1.rs
浏览文件 @
13bc8afa
...
...
@@ -8,8 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// pretty-expanded FIXME #23616
trait
U
{
fn
f
(
self
);
}
impl
U
for
isize
{
fn
f
(
self
)
{}
}
pub
fn
main
()
{
4
.f
();
}
src/test/run-pass/issue-5280.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
type
FontTableTag
=
u32
;
...
...
src/test/run-pass/issue-5321-immediates-with-bare-self.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
trait
Fooable
{
fn
yes
(
self
);
...
...
src/test/run-pass/lexer-crlf-line-endings-string-literal-doc-comment.rs
浏览文件 @
13bc8afa
...
...
@@ -16,7 +16,6 @@
// this directory should enforce it.
// ignore-pretty
// ignore-lexer-test FIXME #15882
/// Doc comment that ends in CRLF
pub
fn
foo
()
{}
...
...
src/test/run-pass/lint-non-camel-case-types-non-uppercase-statics-unicode.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
#![forbid(non_camel_case_types)]
...
...
src/test/run-pass/match-range.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
pub
fn
main
()
{
match
5_u
size
{
...
...
src/test/run-pass/multibyte.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
// Test that multibyte characters don't crash the compiler
pub
fn
main
()
{
...
...
src/test/run-pass/raw-str.rs
浏览文件 @
13bc8afa
无法预览此类型文件
src/test/run-pass/shebang.rs
浏览文件 @
13bc8afa
...
...
@@ -11,6 +11,5 @@
// ignore-pretty: `expand` adds some preludes before shebang
//
// ignore-lexer-test FIXME #15878
pub
fn
main
()
{
println!
(
"Hello World"
);
}
src/test/run-pass/struct-return.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
#[derive(Copy,
Clone)]
pub
struct
Quad
{
a
:
u64
,
b
:
u64
,
c
:
u64
,
d
:
u64
}
...
...
src/test/run-pass/trait-to-str.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15883
trait
to_str
{
...
...
src/test/run-pass/trait-with-bounds-default.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
pub
trait
Clone2
{
/// Returns a copy of the value. The contents of owned pointers
...
...
src/test/run-pass/traits-default-method-self.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
trait
Cat
{
...
...
src/test/run-pass/traits-default-method-trivial.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15877
trait
Cat
{
...
...
src/test/run-pass/unsized.rs
浏览文件 @
13bc8afa
...
...
@@ -10,8 +10,6 @@
// Test syntax checks for `?Sized` syntax.
// pretty-expanded FIXME #23616
use
std
::
marker
::
PhantomData
;
trait
T1
{
}
...
...
src/test/run-pass/unsized2.rs
浏览文件 @
13bc8afa
...
...
@@ -8,8 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// pretty-expanded FIXME #23616
#![allow(unknown_features)]
#![feature(box_syntax)]
...
...
src/test/run-pass/utf8-bom.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
// This file has utf-8 BOM, it should be compiled normally without error.
...
...
src/test/run-pass/utf8.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
// no-pretty-expanded FIXME #15189
pub
fn
main
()
{
...
...
src/test/run-pass/utf8_chars.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
#![feature(collections,
core,
str_char)]
...
...
src/test/run-pass/utf8_idents.rs
浏览文件 @
13bc8afa
...
...
@@ -8,7 +8,6 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15679
#![feature(non_ascii_idents)]
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录