Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
int
Rust
提交
911398b9
R
Rust
项目概览
int
/
Rust
大约 1 年 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
R
Rust
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
911398b9
编写于
8月 14, 2019
作者:
A
Aleksey Kladov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
remove special handling of \r\n from the lexer
上级
004f3ace
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
26 addition
and
104 deletion
+26
-104
src/librustc_lexer/src/lib.rs
src/librustc_lexer/src/lib.rs
+0
-2
src/librustc_lexer/src/unescape.rs
src/librustc_lexer/src/unescape.rs
+8
-28
src/librustc_lexer/src/unescape/tests.rs
src/librustc_lexer/src/unescape/tests.rs
+3
-8
src/libsyntax/parse/lexer/mod.rs
src/libsyntax/parse/lexer/mod.rs
+15
-66
未找到文件。
src/librustc_lexer/src/lib.rs
浏览文件 @
911398b9
...
...
@@ -352,7 +352,6 @@ fn line_comment(&mut self) -> TokenKind {
loop
{
match
self
.nth_char
(
0
)
{
'\n'
=>
break
,
'\r'
if
self
.nth_char
(
1
)
==
'\n'
=>
break
,
EOF_CHAR
if
self
.is_eof
()
=>
break
,
_
=>
{
self
.bump
();
...
...
@@ -525,7 +524,6 @@ fn single_quoted_string(&mut self) -> bool {
match
self
.nth_char
(
0
)
{
'/'
if
!
first
=>
break
,
'\n'
if
self
.nth_char
(
1
)
!=
'\''
=>
break
,
'\r'
if
self
.nth_char
(
1
)
==
'\n'
=>
break
,
EOF_CHAR
if
self
.is_eof
()
=>
break
,
'\''
=>
{
self
.bump
();
...
...
src/librustc_lexer/src/unescape.rs
浏览文件 @
911398b9
...
...
@@ -128,11 +128,7 @@ fn scan_escape(first_char: char, chars: &mut Chars<'_>, mode: Mode) -> Result<ch
if
first_char
!=
'\\'
{
return
match
first_char
{
'\t'
|
'\n'
=>
Err
(
EscapeError
::
EscapeOnlyChar
),
'\r'
=>
Err
(
if
chars
.clone
()
.next
()
==
Some
(
'\n'
)
{
EscapeError
::
EscapeOnlyChar
}
else
{
EscapeError
::
BareCarriageReturn
}),
'\r'
=>
Err
(
EscapeError
::
BareCarriageReturn
),
'\''
if
mode
.in_single_quotes
()
=>
Err
(
EscapeError
::
EscapeOnlyChar
),
'"'
if
mode
.in_double_quotes
()
=>
Err
(
EscapeError
::
EscapeOnlyChar
),
_
=>
{
...
...
@@ -244,27 +240,15 @@ fn unescape_str_or_byte_str<F>(src: &str, mode: Mode, callback: &mut F)
let
unescaped_char
=
match
first_char
{
'\\'
=>
{
let
(
second_char
,
third_char
)
=
{
let
mut
chars
=
chars
.clone
();
(
chars
.next
(),
chars
.next
())
};
match
(
second_char
,
third_char
)
{
(
Some
(
'\n'
),
_
)
|
(
Some
(
'\r'
),
Some
(
'\n'
))
=>
{
let
second_char
=
chars
.clone
()
.next
();
match
second_char
{
Some
(
'\n'
)
=>
{
skip_ascii_whitespace
(
&
mut
chars
);
continue
;
}
_
=>
scan_escape
(
first_char
,
&
mut
chars
,
mode
),
}
}
'\r'
=>
{
let
second_char
=
chars
.clone
()
.next
();
if
second_char
==
Some
(
'\n'
)
{
chars
.next
();
Ok
(
'\n'
)
}
else
{
scan_escape
(
first_char
,
&
mut
chars
,
mode
)
}
}
'\n'
=>
Ok
(
'\n'
),
'\t'
=>
Ok
(
'\t'
),
_
=>
scan_escape
(
first_char
,
&
mut
chars
,
mode
),
...
...
@@ -298,15 +282,11 @@ fn unescape_raw_str_or_byte_str<F>(literal_text: &str, mode: Mode, callback: &mu
while
let
Some
(
curr
)
=
chars
.next
()
{
let
start
=
initial_len
-
chars
.as_str
()
.len
()
-
curr
.len_utf8
();
let
result
=
match
(
curr
,
chars
.clone
()
.next
())
{
(
'\r'
,
Some
(
'\n'
))
=>
{
chars
.next
();
Ok
(
'\n'
)
},
(
'\r'
,
_
)
=>
Err
(
EscapeError
::
BareCarriageReturnInRawString
),
(
c
,
_
)
if
mode
.is_bytes
()
&&
!
c
.is_ascii
()
=>
let
result
=
match
curr
{
'\r'
=>
Err
(
EscapeError
::
BareCarriageReturnInRawString
),
c
if
mode
.is_bytes
()
&&
!
c
.is_ascii
()
=>
Err
(
EscapeError
::
NonAsciiCharInByteString
),
(
c
,
_
)
=>
Ok
(
c
),
c
=>
Ok
(
c
),
};
let
end
=
initial_len
-
chars
.as_str
()
.len
();
...
...
src/librustc_lexer/src/unescape/tests.rs
浏览文件 @
911398b9
...
...
@@ -11,7 +11,6 @@ fn check(literal_text: &str, expected_error: EscapeError) {
check
(
r"\"
,
EscapeError
::
LoneSlash
);
check
(
"
\n
"
,
EscapeError
::
EscapeOnlyChar
);
check
(
"
\r\n
"
,
EscapeError
::
EscapeOnlyChar
);
check
(
"
\t
"
,
EscapeError
::
EscapeOnlyChar
);
check
(
"'"
,
EscapeError
::
EscapeOnlyChar
);
check
(
"
\r
"
,
EscapeError
::
BareCarriageReturn
);
...
...
@@ -31,6 +30,7 @@ fn check(literal_text: &str, expected_error: EscapeError) {
check
(
r"\v"
,
EscapeError
::
InvalidEscape
);
check
(
r"\💩"
,
EscapeError
::
InvalidEscape
);
check
(
r"\●"
,
EscapeError
::
InvalidEscape
);
check
(
"
\\\r
"
,
EscapeError
::
InvalidEscape
);
check
(
r"\x"
,
EscapeError
::
TooShortHexEscape
);
check
(
r"\x0"
,
EscapeError
::
TooShortHexEscape
);
...
...
@@ -116,10 +116,9 @@ fn check(literal_text: &str, expected: &str) {
check
(
"foo"
,
"foo"
);
check
(
""
,
""
);
check
(
"
\t\n
\r\n
"
,
"
\t\n
\n
"
);
check
(
"
\t\n
"
,
"
\t
\n
"
);
check
(
"hello
\\\n
world"
,
"hello world"
);
check
(
"hello
\\\r\n
world"
,
"hello world"
);
check
(
"thread's"
,
"thread's"
)
}
...
...
@@ -134,7 +133,6 @@ fn check(literal_text: &str, expected_error: EscapeError) {
check
(
r"\"
,
EscapeError
::
LoneSlash
);
check
(
"
\n
"
,
EscapeError
::
EscapeOnlyChar
);
check
(
"
\r\n
"
,
EscapeError
::
EscapeOnlyChar
);
check
(
"
\t
"
,
EscapeError
::
EscapeOnlyChar
);
check
(
"'"
,
EscapeError
::
EscapeOnlyChar
);
check
(
"
\r
"
,
EscapeError
::
BareCarriageReturn
);
...
...
@@ -238,10 +236,9 @@ fn check(literal_text: &str, expected: &[u8]) {
check
(
"foo"
,
b
"foo"
);
check
(
""
,
b
""
);
check
(
"
\t\n
\r\n
"
,
b
"
\t\n
\n
"
);
check
(
"
\t\n
"
,
b
"
\t
\n
"
);
check
(
"hello
\\\n
world"
,
b
"hello world"
);
check
(
"hello
\\\r\n
world"
,
b
"hello world"
);
check
(
"thread's"
,
b
"thread's"
)
}
...
...
@@ -253,7 +250,6 @@ fn check(literal: &str, expected: &[(Range<usize>, Result<char, EscapeError>)])
assert_eq!
(
unescaped
,
expected
);
}
check
(
"
\r\n
"
,
&
[(
0
..
2
,
Ok
(
'\n'
))]);
check
(
"
\r
"
,
&
[(
0
..
1
,
Err
(
EscapeError
::
BareCarriageReturnInRawString
))]);
check
(
"
\r
x"
,
&
[(
0
..
1
,
Err
(
EscapeError
::
BareCarriageReturnInRawString
)),
(
1
..
2
,
Ok
(
'x'
))]);
}
...
...
@@ -266,7 +262,6 @@ fn check(literal: &str, expected: &[(Range<usize>, Result<u8, EscapeError>)]) {
assert_eq!
(
unescaped
,
expected
);
}
check
(
"
\r\n
"
,
&
[(
0
..
2
,
Ok
(
byte_from_char
(
'\n'
)))]);
check
(
"
\r
"
,
&
[(
0
..
1
,
Err
(
EscapeError
::
BareCarriageReturnInRawString
))]);
check
(
"🦀"
,
&
[(
0
..
4
,
Err
(
EscapeError
::
NonAsciiCharInByteString
))]);
check
(
...
...
src/libsyntax/parse/lexer/mod.rs
浏览文件 @
911398b9
...
...
@@ -8,9 +8,7 @@
use
rustc_lexer
::
Base
;
use
rustc_lexer
::
unescape
;
use
std
::
borrow
::
Cow
;
use
std
::
char
;
use
std
::
iter
;
use
std
::
convert
::
TryInto
;
use
rustc_data_structures
::
sync
::
Lrc
;
use
log
::
debug
;
...
...
@@ -181,18 +179,7 @@ fn cook_lexer_token(
let
string
=
self
.str_from
(
start
);
// comments with only more "/"s are not doc comments
let
tok
=
if
is_doc_comment
(
string
)
{
let
mut
idx
=
0
;
loop
{
idx
=
match
string
[
idx
..
]
.find
(
'\r'
)
{
None
=>
break
,
Some
(
it
)
=>
idx
+
it
+
1
};
if
string
[
idx
..
]
.chars
()
.next
()
!=
Some
(
'\n'
)
{
self
.err_span_
(
start
+
BytePos
(
idx
as
u32
-
1
),
start
+
BytePos
(
idx
as
u32
),
"bare CR not allowed in doc-comment"
);
}
}
self
.forbid_bare_cr
(
start
,
string
,
"bare CR not allowed in doc-comment"
);
token
::
DocComment
(
Symbol
::
intern
(
string
))
}
else
{
token
::
Comment
...
...
@@ -217,15 +204,10 @@ fn cook_lexer_token(
}
let
tok
=
if
is_doc_comment
{
let
has_cr
=
string
.contains
(
'\r'
);
let
string
=
if
has_cr
{
self
.translate_crlf
(
start
,
string
,
"bare CR not allowed in block doc-comment"
)
}
else
{
string
.into
()
};
token
::
DocComment
(
Symbol
::
intern
(
&
string
[
..
]))
self
.forbid_bare_cr
(
start
,
string
,
"bare CR not allowed in block doc-comment"
);
token
::
DocComment
(
Symbol
::
intern
(
string
))
}
else
{
token
::
Comment
};
...
...
@@ -516,49 +498,16 @@ fn str_from_to(&self, start: BytePos, end: BytePos) -> &str
&
self
.src
[
self
.src_index
(
start
)
..
self
.src_index
(
end
)]
}
/// Converts CRLF to LF in the given string, raising an error on bare CR.
fn
translate_crlf
<
'b
>
(
&
self
,
start
:
BytePos
,
s
:
&
'b
str
,
errmsg
:
&
'b
str
)
->
Cow
<
'b
,
str
>
{
let
mut
chars
=
s
.char_indices
()
.peekable
();
while
let
Some
((
i
,
ch
))
=
chars
.next
()
{
if
ch
==
'\r'
{
if
let
Some
((
lf_idx
,
'\n'
))
=
chars
.peek
()
{
return
translate_crlf_
(
self
,
start
,
s
,
*
lf_idx
,
chars
,
errmsg
)
.into
();
}
let
pos
=
start
+
BytePos
(
i
as
u32
);
let
end_pos
=
start
+
BytePos
((
i
+
ch
.len_utf8
())
as
u32
);
self
.err_span_
(
pos
,
end_pos
,
errmsg
);
}
}
return
s
.into
();
fn
translate_crlf_
(
rdr
:
&
StringReader
<
'_
>
,
start
:
BytePos
,
s
:
&
str
,
mut
j
:
usize
,
mut
chars
:
iter
::
Peekable
<
impl
Iterator
<
Item
=
(
usize
,
char
)
>>
,
errmsg
:
&
str
)
->
String
{
let
mut
buf
=
String
::
with_capacity
(
s
.len
());
// Skip first CR
buf
.push_str
(
&
s
[
..
j
-
1
]);
while
let
Some
((
i
,
ch
))
=
chars
.next
()
{
if
ch
==
'\r'
{
if
j
<
i
{
buf
.push_str
(
&
s
[
j
..
i
]);
}
let
next
=
i
+
ch
.len_utf8
();
j
=
next
;
if
chars
.peek
()
.map
(|(
_
,
ch
)|
*
ch
)
!=
Some
(
'\n'
)
{
let
pos
=
start
+
BytePos
(
i
as
u32
);
let
end_pos
=
start
+
BytePos
(
next
as
u32
);
rdr
.err_span_
(
pos
,
end_pos
,
errmsg
);
}
}
}
if
j
<
s
.len
()
{
buf
.push_str
(
&
s
[
j
..
]);
}
buf
fn
forbid_bare_cr
(
&
self
,
start
:
BytePos
,
s
:
&
str
,
errmsg
:
&
str
)
{
let
mut
idx
=
0
;
loop
{
idx
=
match
s
[
idx
..
]
.find
(
'\r'
)
{
None
=>
break
,
Some
(
it
)
=>
idx
+
it
+
1
};
self
.err_span_
(
start
+
BytePos
(
idx
as
u32
-
1
),
start
+
BytePos
(
idx
as
u32
),
errmsg
);
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录