提交 ca2a50fa 编写于 作者: V Vadim Petrochenkov

syntax: Turn `token::Lit` into a struct

上级 558559e7
......@@ -1249,8 +1249,7 @@ fn print_expr_addr_of(&mut self,
fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
self.maybe_print_comment(lit.span.lo())?;
let (token, suffix) = lit.node.to_lit_token();
self.writer().word(pprust::literal_to_string(token, suffix))
self.writer().word(pprust::literal_to_string(lit.node.to_lit_token()))
}
pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> {
......
......@@ -165,7 +165,6 @@ fn hash_stable<W: StableHasherResult>(&self,
impl_stable_hash_for!(struct ::syntax::ast::Lit {
node,
token,
suffix,
span
});
......@@ -288,17 +287,23 @@ fn hash_stable<W: StableHasherResult>(&self,
}
}
impl_stable_hash_for!(enum token::Lit {
Bool(val),
Byte(val),
Char(val),
Err(val),
Integer(val),
Float(val),
Str_(val),
ByteStr(val),
StrRaw(val, n),
ByteStrRaw(val, n)
impl_stable_hash_for!(enum token::LitKind {
Bool,
Byte,
Char,
Integer,
Float,
Str,
ByteStr,
StrRaw(n),
ByteStrRaw(n),
Err
});
impl_stable_hash_for!(struct token::Lit {
kind,
symbol,
suffix
});
fn hash_token<'a, 'gcx, W: StableHasherResult>(
......@@ -348,10 +353,7 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
token::Token::CloseDelim(delim_token) => {
std_hash::Hash::hash(&delim_token, hasher);
}
token::Token::Literal(lit, opt_name) => {
lit.hash_stable(hcx, hasher);
opt_name.hash_stable(hcx, hasher);
}
token::Token::Literal(lit) => lit.hash_stable(hcx, hasher),
token::Token::Ident(ident, is_raw) => {
ident.name.hash_stable(hcx, hasher);
......
......@@ -310,17 +310,17 @@ fn write_token<W: Writer>(&mut self,
}
}
token::Literal(lit, _suf) => {
match lit {
token::Literal(lit) => {
match lit.kind {
// Text literals.
token::Byte(..) | token::Char(..) | token::Err(..) |
token::ByteStr(..) | token::ByteStrRaw(..) |
token::Str_(..) | token::StrRaw(..) => Class::String,
token::Byte | token::Char | token::Err |
token::ByteStr | token::ByteStrRaw(..) |
token::Str | token::StrRaw(..) => Class::String,
// Number literals.
token::Integer(..) | token::Float(..) => Class::Number,
token::Integer | token::Float => Class::Number,
token::Bool(..) => panic!("literal token contains `Lit::Bool`"),
token::Bool => panic!("literal token contains `Lit::Bool`"),
}
}
......
......@@ -1347,8 +1347,6 @@ pub enum StrStyle {
pub struct Lit {
/// The original literal token as written in source code.
pub token: token::Lit,
/// The original literal suffix as written in source code.
pub suffix: Option<Symbol>,
/// The "semantic" representation of the literal lowered from the original tokens.
/// Strings are unescaped, hexadecimal forms are eliminated, etc.
/// FIXME: Remove this and only create the semantic representation during lowering to HIR.
......
......@@ -77,8 +77,8 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
},
(3, Some(&TokenTree::Token(_, token::Ident(ref code, _))),
Some(&TokenTree::Token(_, token::Comma)),
Some(&TokenTree::Token(_, token::Literal(token::StrRaw(description, _), None)))) => {
(code, Some(description))
Some(&TokenTree::Token(_, token::Literal(token::Lit { symbol, .. })))) => {
(code, Some(symbol))
}
_ => unreachable!()
};
......
use crate::ast::{self, Ident};
use crate::parse::{token, ParseSess};
use crate::parse::ParseSess;
use crate::parse::token::{self, Token};
use crate::symbol::Symbol;
use crate::parse::unescape;
use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char};
......@@ -21,7 +22,7 @@
#[derive(Clone, Debug)]
pub struct TokenAndSpan {
pub tok: token::Token,
pub tok: Token,
pub sp: Span,
}
......@@ -55,7 +56,7 @@ pub struct StringReader<'a> {
/// Stop reading src at this index.
crate end_src_index: usize,
// cached:
peek_tok: token::Token,
peek_tok: Token,
peek_span: Span,
peek_span_src_raw: Span,
fatal_errs: Vec<DiagnosticBuilder<'a>>,
......@@ -726,7 +727,7 @@ fn scan_digits(&mut self, real_radix: u32, scan_radix: u32) -> usize {
}
/// Lex a LIT_INTEGER or a LIT_FLOAT
fn scan_number(&mut self, c: char) -> token::Lit {
fn scan_number(&mut self, c: char) -> (token::LitKind, Symbol) {
let mut base = 10;
let start_bpos = self.pos;
self.bump();
......@@ -753,7 +754,7 @@ fn scan_number(&mut self, c: char) -> token::Lit {
}
_ => {
// just a 0
return token::Integer(self.name_from(start_bpos));
return (token::Integer, self.name_from(start_bpos));
}
}
} else if c.is_digit(10) {
......@@ -765,7 +766,7 @@ fn scan_number(&mut self, c: char) -> token::Lit {
if num_digits == 0 {
self.err_span_(start_bpos, self.pos, "no valid digits found for number");
return token::Integer(Symbol::intern("0"));
return (token::Integer, Symbol::intern("0"));
}
// might be a float, but don't be greedy if this is actually an
......@@ -783,17 +784,17 @@ fn scan_number(&mut self, c: char) -> token::Lit {
let pos = self.pos;
self.check_float_base(start_bpos, pos, base);
token::Float(self.name_from(start_bpos))
(token::Float, self.name_from(start_bpos))
} else {
// it might be a float if it has an exponent
if self.ch_is('e') || self.ch_is('E') {
self.scan_float_exponent();
let pos = self.pos;
self.check_float_base(start_bpos, pos, base);
return token::Float(self.name_from(start_bpos));
return (token::Float, self.name_from(start_bpos));
}
// but we certainly have an integer!
token::Integer(self.name_from(start_bpos))
(token::Integer, self.name_from(start_bpos))
}
}
......@@ -846,7 +847,7 @@ fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: us
}
}
fn binop(&mut self, op: token::BinOpToken) -> token::Token {
fn binop(&mut self, op: token::BinOpToken) -> Token {
self.bump();
if self.ch_is('=') {
self.bump();
......@@ -858,7 +859,7 @@ fn binop(&mut self, op: token::BinOpToken) -> token::Token {
/// Returns the next token from the string, advances the input past that
/// token, and updates the interner
fn next_token_inner(&mut self) -> Result<token::Token, ()> {
fn next_token_inner(&mut self) -> Result<Token, ()> {
let c = self.ch;
if ident_start(c) {
......@@ -912,10 +913,10 @@ fn next_token_inner(&mut self) -> Result<token::Token, ()> {
}
if is_dec_digit(c) {
let num = self.scan_number(c.unwrap());
let (kind, symbol) = self.scan_number(c.unwrap());
let suffix = self.scan_optional_raw_name();
debug!("next_token_inner: scanned number {:?}, {:?}", num, suffix);
return Ok(token::Literal(num, suffix));
debug!("next_token_inner: scanned number {:?}, {:?}, {:?}", kind, symbol, suffix);
return Ok(Token::lit(kind, symbol, suffix));
}
match c.expect("next_token_inner called at EOF") {
......@@ -1073,10 +1074,10 @@ fn next_token_inner(&mut self) -> Result<token::Token, ()> {
// lifetimes shouldn't end with a single quote
// if we find one, then this is an invalid character literal
if self.ch_is('\'') {
let id = self.name_from(start);
let symbol = self.name_from(start);
self.bump();
self.validate_char_escape(start_with_quote);
return Ok(token::Literal(token::Char(id), None))
return Ok(Token::lit(token::Char, symbol, None));
}
// Include the leading `'` in the real identifier, for macro
......@@ -1098,43 +1099,43 @@ fn next_token_inner(&mut self) -> Result<token::Token, ()> {
return Ok(token::Lifetime(ident));
}
let msg = "unterminated character literal";
let id = self.scan_single_quoted_string(start_with_quote, msg);
let symbol = self.scan_single_quoted_string(start_with_quote, msg);
self.validate_char_escape(start_with_quote);
let suffix = self.scan_optional_raw_name();
Ok(token::Literal(token::Char(id), suffix))
Ok(Token::lit(token::Char, symbol, suffix))
}
'b' => {
self.bump();
let lit = match self.ch {
let (kind, symbol) = match self.ch {
Some('\'') => {
let start_with_quote = self.pos;
self.bump();
let msg = "unterminated byte constant";
let id = self.scan_single_quoted_string(start_with_quote, msg);
let symbol = self.scan_single_quoted_string(start_with_quote, msg);
self.validate_byte_escape(start_with_quote);
token::Byte(id)
(token::Byte, symbol)
},
Some('"') => {
let start_with_quote = self.pos;
let msg = "unterminated double quote byte string";
let id = self.scan_double_quoted_string(msg);
let symbol = self.scan_double_quoted_string(msg);
self.validate_byte_str_escape(start_with_quote);
token::ByteStr(id)
(token::ByteStr, symbol)
},
Some('r') => self.scan_raw_byte_string(),
_ => unreachable!(), // Should have been a token::Ident above.
};
let suffix = self.scan_optional_raw_name();
Ok(token::Literal(lit, suffix))
Ok(Token::lit(kind, symbol, suffix))
}
'"' => {
let start_with_quote = self.pos;
let msg = "unterminated double quote string";
let id = self.scan_double_quoted_string(msg);
let symbol = self.scan_double_quoted_string(msg);
self.validate_str_escape(start_with_quote);
let suffix = self.scan_optional_raw_name();
Ok(token::Literal(token::Str_(id), suffix))
Ok(Token::lit(token::Str, symbol, suffix))
}
'r' => {
let start_bpos = self.pos;
......@@ -1205,14 +1206,14 @@ fn next_token_inner(&mut self) -> Result<token::Token, ()> {
}
self.bump();
let id = if valid {
let symbol = if valid {
self.name_from_to(content_start_bpos, content_end_bpos)
} else {
Symbol::intern("??")
};
let suffix = self.scan_optional_raw_name();
Ok(token::Literal(token::StrRaw(id, hash_count), suffix))
Ok(Token::lit(token::StrRaw(hash_count), symbol, suffix))
}
'-' => {
if self.nextch_is('>') {
......@@ -1366,7 +1367,7 @@ fn scan_double_quoted_string(&mut self, unterminated_msg: &str) -> ast::Name {
id
}
fn scan_raw_byte_string(&mut self) -> token::Lit {
fn scan_raw_byte_string(&mut self) -> (token::LitKind, Symbol) {
let start_bpos = self.pos;
self.bump();
let mut hash_count = 0;
......@@ -1423,7 +1424,7 @@ fn scan_raw_byte_string(&mut self) -> token::Lit {
self.bump();
token::ByteStrRaw(self.name_from_to(content_start_bpos, content_end_bpos), hash_count)
(token::ByteStrRaw(hash_count), self.name_from_to(content_start_bpos, content_end_bpos))
}
fn validate_char_escape(&self, start_with_quote: BytePos) {
......@@ -1637,15 +1638,19 @@ fn t1() {
// check that the given reader produces the desired stream
// of tokens (stop checking after exhausting the expected vec)
fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<token::Token>) {
fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<Token>) {
for expected_tok in &expected {
assert_eq!(&string_reader.next_token().tok, expected_tok);
}
}
// make the identifier by looking up the string in the interner
fn mk_ident(id: &str) -> token::Token {
token::Token::from_ast_ident(Ident::from_str(id))
fn mk_ident(id: &str) -> Token {
Token::from_ast_ident(Ident::from_str(id))
}
fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> Token {
Token::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
}
#[test]
......@@ -1694,7 +1699,7 @@ fn character_a() {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token().tok,
token::Literal(token::Char(Symbol::intern("a")), None));
mk_lit(token::Char, "a", None));
})
}
......@@ -1704,7 +1709,7 @@ fn character_space() {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token().tok,
token::Literal(token::Char(Symbol::intern(" ")), None));
mk_lit(token::Char, " ", None));
})
}
......@@ -1714,7 +1719,7 @@ fn character_escaped() {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token().tok,
token::Literal(token::Char(Symbol::intern("\\n")), None));
mk_lit(token::Char, "\\n", None));
})
}
......@@ -1724,7 +1729,7 @@ fn lifetime_name() {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token().tok,
token::Lifetime(Ident::from_str("'abc")));
token::Lifetime(Ident::from_str("'abc")));
})
}
......@@ -1733,10 +1738,8 @@ fn raw_string() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string())
.next_token()
.tok,
token::Literal(token::StrRaw(Symbol::intern("\"#a\\b\x00c\""), 3), None));
assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token().tok,
mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None));
})
}
......@@ -1748,18 +1751,16 @@ fn literal_suffixes() {
macro_rules! test {
($input: expr, $tok_type: ident, $tok_contents: expr) => {{
assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token().tok,
token::Literal(token::$tok_type(Symbol::intern($tok_contents)),
Some(Symbol::intern("suffix"))));
mk_lit(token::$tok_type, $tok_contents, Some("suffix")));
// with a whitespace separator:
assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token().tok,
token::Literal(token::$tok_type(Symbol::intern($tok_contents)),
None));
mk_lit(token::$tok_type, $tok_contents, None));
}}
}
test!("'a'", Char, "a");
test!("b'a'", Byte, "a");
test!("\"a\"", Str_, "a");
test!("\"a\"", Str, "a");
test!("b\"a\"", ByteStr, "a");
test!("1234", Integer, "1234");
test!("0b101", Integer, "0b101");
......@@ -1768,14 +1769,11 @@ fn literal_suffixes() {
test!("1.0e10", Float, "1.0e10");
assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token().tok,
token::Literal(token::Integer(Symbol::intern("2")),
Some(Symbol::intern("us"))));
mk_lit(token::Integer, "2", Some("us")));
assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token().tok,
token::Literal(token::StrRaw(Symbol::intern("raw"), 3),
Some(Symbol::intern("suffix"))));
mk_lit(token::StrRaw(3), "raw", Some("suffix")));
assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token().tok,
token::Literal(token::ByteStrRaw(Symbol::intern("raw"), 3),
Some(Symbol::intern("suffix"))));
mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")));
})
}
......@@ -1796,8 +1794,7 @@ fn nested_block_comments() {
token::Comment => {}
_ => panic!("expected a comment!"),
}
assert_eq!(lexer.next_token().tok,
token::Literal(token::Char(Symbol::intern("a")), None));
assert_eq!(lexer.next_token().tok, mk_lit(token::Char, "a", None));
})
}
......
......@@ -3,7 +3,7 @@
use crate::ast::{self, Ident, Lit, LitKind};
use crate::parse::parser::Parser;
use crate::parse::PResult;
use crate::parse::token;
use crate::parse::token::{self, Token};
use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte};
use crate::print::pprust;
use crate::symbol::{kw, Symbol};
......@@ -27,14 +27,21 @@
}
impl LitError {
crate fn report(&self, diag: &Handler, lit: token::Lit, suf: Option<Symbol>, span: Span) {
crate fn report(
&self,
diag: &Handler,
token::Lit { kind, suffix, .. }: token::Lit,
span: Span,
) {
match *self {
LitError::NotLiteral | LitError::LexerError => {}
LitError::InvalidSuffix => {
expect_no_suffix(diag, span, &format!("{} {}", lit.article(), lit.descr()), suf);
expect_no_suffix(
diag, span, &format!("{} {}", kind.article(), kind.descr()), suffix
);
}
LitError::InvalidIntSuffix => {
let suf = suf.expect("suffix error with no suffix").as_str();
let suf = suffix.expect("suffix error with no suffix").as_str();
if looks_like_width_suffix(&['i', 'u'], &suf) {
// If it looks like a width, try to be helpful.
let msg = format!("invalid width `{}` for integer literal", &suf[1..]);
......@@ -50,7 +57,7 @@ impl LitError {
}
}
LitError::InvalidFloatSuffix => {
let suf = suf.expect("suffix error with no suffix").as_str();
let suf = suffix.expect("suffix error with no suffix").as_str();
if looks_like_width_suffix(&['f'], &suf) {
// If it looks like a width, try to be helpful.
let msg = format!("invalid width `{}` for float literal", &suf[1..]);
......@@ -84,43 +91,42 @@ impl LitKind {
/// If diagnostic handler is passed, always returns `Some`,
/// possibly after reporting non-fatal errors and recovery.
fn from_lit_token(
lit: token::Lit,
suf: Option<Symbol>,
token::Lit { kind, symbol, suffix }: token::Lit,
) -> Result<LitKind, LitError> {
if suf.is_some() && !lit.may_have_suffix() {
if suffix.is_some() && !kind.may_have_suffix() {
return Err(LitError::InvalidSuffix);
}
Ok(match lit {
token::Bool(i) => {
assert!(i == kw::True || i == kw::False);
LitKind::Bool(i == kw::True)
Ok(match kind {
token::Bool => {
assert!(symbol == kw::True || symbol == kw::False);
LitKind::Bool(symbol == kw::True)
}
token::Byte(i) => {
match unescape_byte(&i.as_str()) {
token::Byte => {
match unescape_byte(&symbol.as_str()) {
Ok(c) => LitKind::Byte(c),
Err(_) => LitKind::Err(i),
Err(_) => return Err(LitError::LexerError),
}
},
token::Char(i) => {
match unescape_char(&i.as_str()) {
token::Char => {
match unescape_char(&symbol.as_str()) {
Ok(c) => LitKind::Char(c),
Err(_) => return Err(LitError::LexerError),
}
},
token::Err(i) => LitKind::Err(i),
// There are some valid suffixes for integer and float literals,
// so all the handling is done internally.
token::Integer(s) => return integer_lit(s, suf),
token::Float(s) => return float_lit(s, suf),
token::Integer => return integer_lit(symbol, suffix),
token::Float => return float_lit(symbol, suffix),
token::Str_(mut sym) => {
token::Str => {
// If there are no characters requiring special treatment we can
// reuse the symbol from the token. Otherwise, we must generate a
// new symbol because the string in the LitKind is different to the
// string in the token.
let mut error = None;
let mut sym = symbol;
let s = &sym.as_str();
if s.as_bytes().iter().any(|&c| c == b'\\' || c == b'\r') {
let mut buf = String::with_capacity(s.len());
......@@ -138,16 +144,17 @@ fn from_lit_token(
LitKind::Str(sym, ast::StrStyle::Cooked)
}
token::StrRaw(mut sym, n) => {
token::StrRaw(n) => {
// Ditto.
let mut sym = symbol;
let s = &sym.as_str();
if s.contains('\r') {
sym = Symbol::intern(&raw_str_lit(s));
}
LitKind::Str(sym, ast::StrStyle::Raw(n))
}
token::ByteStr(i) => {
let s = &i.as_str();
token::ByteStr => {
let s = &symbol.as_str();
let mut buf = Vec::with_capacity(s.len());
let mut error = None;
unescape_byte_str(s, &mut |_, unescaped_byte| {
......@@ -162,36 +169,37 @@ fn from_lit_token(
buf.shrink_to_fit();
LitKind::ByteStr(Lrc::new(buf))
}
token::ByteStrRaw(i, _) => {
LitKind::ByteStr(Lrc::new(i.to_string().into_bytes()))
token::ByteStrRaw(_) => {
LitKind::ByteStr(Lrc::new(symbol.to_string().into_bytes()))
}
token::Err => LitKind::Err(symbol),
})
}
/// Attempts to recover a token from semantic literal.
/// This function is used when the original token doesn't exist (e.g. the literal is created
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
pub fn to_lit_token(&self) -> (token::Lit, Option<Symbol>) {
match *self {
pub fn to_lit_token(&self) -> token::Lit {
let (kind, symbol, suffix) = match *self {
LitKind::Str(string, ast::StrStyle::Cooked) => {
let escaped = string.as_str().escape_default().to_string();
(token::Lit::Str_(Symbol::intern(&escaped)), None)
(token::Str, Symbol::intern(&escaped), None)
}
LitKind::Str(string, ast::StrStyle::Raw(n)) => {
(token::Lit::StrRaw(string, n), None)
(token::StrRaw(n), string, None)
}
LitKind::ByteStr(ref bytes) => {
let string = bytes.iter().cloned().flat_map(ascii::escape_default)
.map(Into::<char>::into).collect::<String>();
(token::Lit::ByteStr(Symbol::intern(&string)), None)
(token::ByteStr, Symbol::intern(&string), None)
}
LitKind::Byte(byte) => {
let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
(token::Lit::Byte(Symbol::intern(&string)), None)
(token::Byte, Symbol::intern(&string), None)
}
LitKind::Char(ch) => {
let string: String = ch.escape_default().map(Into::<char>::into).collect();
(token::Lit::Char(Symbol::intern(&string)), None)
(token::Char, Symbol::intern(&string), None)
}
LitKind::Int(n, ty) => {
let suffix = match ty {
......@@ -199,29 +207,33 @@ pub fn to_lit_token(&self) -> (token::Lit, Option<Symbol>) {
ast::LitIntType::Signed(ty) => Some(Symbol::intern(ty.ty_to_string())),
ast::LitIntType::Unsuffixed => None,
};
(token::Lit::Integer(Symbol::intern(&n.to_string())), suffix)
(token::Integer, Symbol::intern(&n.to_string()), suffix)
}
LitKind::Float(symbol, ty) => {
(token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string())))
(token::Float, symbol, Some(Symbol::intern(ty.ty_to_string())))
}
LitKind::FloatUnsuffixed(symbol) => {
(token::Float, symbol, None)
}
LitKind::FloatUnsuffixed(symbol) => (token::Lit::Float(symbol), None),
LitKind::Bool(value) => {
let kw = if value { kw::True } else { kw::False };
(token::Lit::Bool(kw), None)
let symbol = if value { kw::True } else { kw::False };
(token::Bool, symbol, None)
}
LitKind::Err(val) => (token::Lit::Err(val), None),
}
LitKind::Err(symbol) => {
(token::Err, symbol, None)
}
};
token::Lit::new(kind, symbol, suffix)
}
}
impl Lit {
fn from_lit_token(
token: token::Lit,
suffix: Option<Symbol>,
span: Span,
) -> Result<Lit, LitError> {
let node = LitKind::from_lit_token(token, suffix)?;
Ok(Lit { node, token, suffix, span })
Ok(Lit { token, node: LitKind::from_lit_token(token)?, span })
}
/// Converts literal token with a suffix into an AST literal.
......@@ -232,11 +244,11 @@ fn from_lit_token(
token: &token::Token,
span: Span,
) -> Result<Lit, LitError> {
let (lit, suf) = match *token {
let lit = match *token {
token::Ident(ident, false) if ident.name == kw::True || ident.name == kw::False =>
(token::Bool(ident.name), None),
token::Literal(token, suffix) =>
(token, suffix),
token::Lit::new(token::Bool, ident.name, None),
token::Literal(lit) =>
lit,
token::Interpolated(ref nt) => {
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
if let ast::ExprKind::Lit(lit) = &expr.node {
......@@ -248,22 +260,21 @@ fn from_lit_token(
_ => return Err(LitError::NotLiteral)
};
Lit::from_lit_token(lit, suf, span)
Lit::from_lit_token(lit, span)
}
/// Attempts to recover an AST literal from semantic literal.
/// This function is used when the original token doesn't exist (e.g. the literal is created
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
pub fn from_lit_kind(node: LitKind, span: Span) -> Lit {
let (token, suffix) = node.to_lit_token();
Lit { node, token, suffix, span }
Lit { token: node.to_lit_token(), node, span }
}
/// Losslessly convert an AST literal into a token stream.
crate fn tokens(&self) -> TokenStream {
let token = match self.token {
token::Bool(symbol) => token::Ident(Ident::new(symbol, self.span), false),
token => token::Literal(token, self.suffix),
let token = match self.token.kind {
token::Bool => token::Ident(Ident::new(self.token.symbol, self.span), false),
_ => token::Literal(self.token),
};
TokenTree::Token(self.span, token).into()
}
......@@ -276,11 +287,11 @@ impl<'a> Parser<'a> {
if self.token == token::Dot {
// Attempt to recover `.4` as `0.4`.
recovered = self.look_ahead(1, |t| {
if let token::Literal(token::Integer(val), suf) = *t {
if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = *t {
let next_span = self.look_ahead_span(1);
if self.span.hi() == next_span.lo() {
let sym = String::from("0.") + &val.as_str();
let token = token::Literal(token::Float(Symbol::intern(&sym)), suf);
let s = String::from("0.") + &symbol.as_str();
let token = Token::lit(token::Float, Symbol::intern(&s), suffix);
return Some((token, self.span.to(next_span)));
}
}
......@@ -313,10 +324,11 @@ impl<'a> Parser<'a> {
return Err(self.span_fatal(span, &msg));
}
Err(err) => {
let (lit, suf) = token.expect_lit();
let lit = token.expect_lit();
self.bump();
err.report(&self.sess.span_diagnostic, lit, suf, span);
return Ok(Lit::from_lit_token(token::Err(lit.symbol()), suf, span).ok().unwrap());
err.report(&self.sess.span_diagnostic, lit, span);
let lit = token::Lit::new(token::Err, lit.symbol, lit.suffix);
return Ok(Lit::from_lit_token(lit, span).ok().unwrap());
}
}
}
......
......@@ -352,10 +352,12 @@ fn next_desugared(&mut self) -> TokenAndSpan {
let body = TokenTree::Delimited(
delim_span,
token::Bracket,
[TokenTree::Token(sp, token::Ident(ast::Ident::with_empty_ctxt(sym::doc), false)),
TokenTree::Token(sp, token::Eq),
TokenTree::Token(sp, token::Literal(
token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))
[
TokenTree::Token(sp, token::Ident(ast::Ident::with_empty_ctxt(sym::doc), false)),
TokenTree::Token(sp, token::Eq),
TokenTree::Token(sp, token::Token::lit(
token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None
)),
]
.iter().cloned().collect::<TokenStream>().into(),
);
......@@ -2241,10 +2243,10 @@ fn parse_mutability(&mut self) -> Mutability {
}
fn parse_field_name(&mut self) -> PResult<'a, Ident> {
if let token::Literal(token::Integer(name), suffix) = self.token {
if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = self.token {
self.expect_no_suffix(self.span, "a tuple index", suffix);
self.bump();
Ok(Ident::new(name, self.prev_span))
Ok(Ident::new(symbol, self.prev_span))
} else {
self.parse_ident_common(false)
}
......@@ -3045,19 +3047,19 @@ fn parse_dot_or_call_expr_with_(&mut self, e0: P<Expr>, lo: Span) -> PResult<'a,
token::Ident(..) => {
e = self.parse_dot_suffix(e, lo)?;
}
token::Literal(token::Integer(name), suffix) => {
token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) => {
let span = self.span;
self.bump();
let field = ExprKind::Field(e, Ident::new(name, span));
let field = ExprKind::Field(e, Ident::new(symbol, span));
e = self.mk_expr(lo.to(span), field, ThinVec::new());
self.expect_no_suffix(span, "a tuple index", suffix);
}
token::Literal(token::Float(n), _suf) => {
token::Literal(token::Lit { kind: token::Float, symbol, .. }) => {
self.bump();
let fstr = n.as_str();
let mut err = self.diagnostic()
.struct_span_err(self.prev_span, &format!("unexpected token: `{}`", n));
let fstr = symbol.as_str();
let msg = format!("unexpected token: `{}`", symbol);
let mut err = self.diagnostic().struct_span_err(self.prev_span, &msg);
err.span_label(self.prev_span, "unexpected token");
if fstr.chars().all(|x| "0123456789.".contains(x)) {
let float = match fstr.parse::<f64>().ok() {
......@@ -7557,11 +7559,12 @@ fn parse_item_enum(&mut self) -> PResult<'a, ItemInfo> {
/// the `extern` keyword, if one is found.
fn parse_opt_abi(&mut self) -> PResult<'a, Option<Abi>> {
match self.token {
token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(s, _), suf) => {
token::Literal(token::Lit { kind: token::Str, symbol, suffix }) |
token::Literal(token::Lit { kind: token::StrRaw(..), symbol, suffix }) => {
let sp = self.span;
self.expect_no_suffix(sp, "an ABI spec", suf);
self.expect_no_suffix(sp, "an ABI spec", suffix);
self.bump();
match abi::lookup(&s.as_str()) {
match abi::lookup(&symbol.as_str()) {
Some(abi) => Ok(Some(abi)),
None => {
let prev_span = self.prev_span;
......@@ -7570,7 +7573,7 @@ fn parse_opt_abi(&mut self) -> PResult<'a, Option<Abi>> {
prev_span,
E0703,
"invalid ABI: found `{}`",
s);
symbol);
err.span_label(prev_span, "invalid ABI");
err.help(&format!("valid ABIs: {}", abi::all_names().join(", ")));
err.emit();
......@@ -8370,8 +8373,10 @@ pub fn parse_crate_mod(&mut self) -> PResult<'a, Crate> {
pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> {
let ret = match self.token {
token::Literal(token::Str_(s), suf) => (s, ast::StrStyle::Cooked, suf),
token::Literal(token::StrRaw(s, n), suf) => (s, ast::StrStyle::Raw(n), suf),
token::Literal(token::Lit { kind: token::Str, symbol, suffix }) =>
(symbol, ast::StrStyle::Cooked, suffix),
token::Literal(token::Lit { kind: token::StrRaw(n), symbol, suffix }) =>
(symbol, ast::StrStyle::Raw(n), suffix),
_ => return None
};
self.bump();
......
pub use BinOpToken::*;
pub use Nonterminal::*;
pub use DelimToken::*;
pub use Lit::*;
pub use LitKind::*;
pub use Token::*;
use crate::ast::{self};
......@@ -59,59 +59,62 @@ pub fn is_empty(self) -> bool {
}
}
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
pub enum Lit {
Bool(ast::Name), // AST only, must never appear in a `Token`
Byte(ast::Name),
Char(ast::Name),
Err(ast::Name),
Integer(ast::Name),
Float(ast::Name),
Str_(ast::Name),
StrRaw(ast::Name, u16), /* raw str delimited by n hash symbols */
ByteStr(ast::Name),
ByteStrRaw(ast::Name, u16), /* raw byte str delimited by n hash symbols */
#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)]
pub enum LitKind {
Bool, // AST only, must never appear in a `Token`
Byte,
Char,
Integer,
Float,
Str,
StrRaw(u16), // raw string delimited by `n` hash symbols
ByteStr,
ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols
Err,
}
#[cfg(target_arch = "x86_64")]
static_assert_size!(Lit, 8);
impl Lit {
crate fn symbol(&self) -> Symbol {
match *self {
Bool(s) | Byte(s) | Char(s) | Integer(s) | Float(s) | Err(s) |
Str_(s) | StrRaw(s, _) | ByteStr(s) | ByteStrRaw(s, _) => s
}
}
#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)]
pub struct Lit {
pub kind: LitKind,
pub symbol: Symbol,
pub suffix: Option<Symbol>,
}
crate fn article(&self) -> &'static str {
match *self {
Integer(_) | Err(_) => "an",
impl LitKind {
crate fn article(self) -> &'static str {
match self {
Integer | Err => "an",
_ => "a",
}
}
crate fn descr(&self) -> &'static str {
match *self {
Bool(_) => panic!("literal token contains `Lit::Bool`"),
Byte(_) => "byte literal",
Char(_) => "char literal",
Err(_) => "invalid literal",
Integer(_) => "integer literal",
Float(_) => "float literal",
Str_(_) | StrRaw(..) => "string literal",
ByteStr(_) | ByteStrRaw(..) => "byte string literal"
crate fn descr(self) -> &'static str {
match self {
Bool => panic!("literal token contains `Lit::Bool`"),
Byte => "byte literal",
Char => "char literal",
Integer => "integer literal",
Float => "float literal",
Str | StrRaw(..) => "string literal",
ByteStr | ByteStrRaw(..) => "byte string literal",
Err => "invalid literal",
}
}
crate fn may_have_suffix(&self) -> bool {
match *self {
Integer(..) | Float(..) | Err(..) => true,
crate fn may_have_suffix(self) -> bool {
match self {
Integer | Float | Err => true,
_ => false,
}
}
}
impl Lit {
pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
Lit { kind, symbol, suffix }
}
}
pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool {
let ident_token: Token = Ident(ident, is_raw);
......@@ -201,7 +204,7 @@ pub enum Token {
CloseDelim(DelimToken),
/* Literals */
Literal(Lit, Option<ast::Name>),
Literal(Lit),
/* Name components */
Ident(ast::Ident, /* is_raw */ bool),
......@@ -318,6 +321,10 @@ pub fn can_begin_const_arg(&self) -> bool {
self == &Question || self == &OpenDelim(Paren)
}
pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Token {
Literal(Lit::new(kind, symbol, suffix))
}
/// Returns `true` if the token is any literal
crate fn is_lit(&self) -> bool {
match *self {
......@@ -326,9 +333,9 @@ pub fn can_begin_const_arg(&self) -> bool {
}
}
crate fn expect_lit(&self) -> (Lit, Option<Symbol>) {
crate fn expect_lit(&self) -> Lit {
match *self {
Literal(lit, suf) => (lit, suf),
Literal(lit) => lit,
_=> panic!("`expect_lit` called on non-literal"),
}
}
......@@ -579,13 +586,13 @@ pub fn is_reserved_ident(&self) -> bool {
(&DocComment(a), &DocComment(b)) |
(&Shebang(a), &Shebang(b)) => a == b,
(&Literal(a), &Literal(b)) => a == b,
(&Lifetime(a), &Lifetime(b)) => a.name == b.name,
(&Ident(a, b), &Ident(c, d)) => b == d && (a.name == c.name ||
a.name == kw::DollarCrate ||
c.name == kw::DollarCrate),
(&Literal(a, b), &Literal(c, d)) => b == d && a == c,
(&Interpolated(_), &Interpolated(_)) => false,
_ => panic!("forgot to add a token?"),
......
......@@ -163,22 +163,22 @@ fn binop_to_string(op: BinOpToken) -> &'static str {
}
}
pub fn literal_to_string(lit: token::Lit, suffix: Option<ast::Name>) -> String {
let mut out = match lit {
token::Byte(b) => format!("b'{}'", b),
token::Char(c) => format!("'{}'", c),
token::Err(c) => format!("'{}'", c),
token::Bool(c) |
token::Float(c) |
token::Integer(c) => c.to_string(),
token::Str_(s) => format!("\"{}\"", s),
token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}",
delim="#".repeat(n as usize),
string=s),
token::ByteStr(v) => format!("b\"{}\"", v),
token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}",
delim="#".repeat(n as usize),
string=s),
pub fn literal_to_string(token::Lit { kind, symbol, suffix }: token::Lit) -> String {
let mut out = match kind {
token::Byte => format!("b'{}'", symbol),
token::Char => format!("'{}'", symbol),
token::Bool |
token::Float |
token::Integer => symbol.to_string(),
token::Str => format!("\"{}\"", symbol),
token::StrRaw(n) => format!("r{delim}\"{string}\"{delim}",
delim="#".repeat(n as usize),
string=symbol),
token::ByteStr => format!("b\"{}\"", symbol),
token::ByteStrRaw(n) => format!("br{delim}\"{string}\"{delim}",
delim="#".repeat(n as usize),
string=symbol),
token::Err => format!("'{}'", symbol),
};
if let Some(suffix) = suffix {
......@@ -231,7 +231,7 @@ pub fn token_to_string(tok: &Token) -> String {
token::SingleQuote => "'".to_string(),
/* Literals */
token::Literal(lit, suf) => literal_to_string(lit, suf),
token::Literal(lit) => literal_to_string(lit),
/* Name components */
token::Ident(s, false) => s.to_string(),
......@@ -571,7 +571,7 @@ fn next_comment(&mut self) -> Option<comments::Comment> {
fn print_literal(&mut self, lit: &ast::Lit) -> io::Result<()> {
self.maybe_print_comment(lit.span.lo())?;
self.writer().word(literal_to_string(lit.token, lit.suffix))
self.writer().word(literal_to_string(lit.token))
}
fn print_string(&mut self, st: &str,
......
......@@ -4,7 +4,7 @@
use syntax::source_map::Spanned;
use syntax::ext::base::*;
use syntax::ext::build::AstBuilder;
use syntax::parse::token;
use syntax::parse::token::{self, Token};
use syntax::parse::parser::Parser;
use syntax::print::pprust;
use syntax::ptr::P;
......@@ -31,13 +31,10 @@ pub fn expand_assert<'cx>(
tts: custom_message.unwrap_or_else(|| {
TokenStream::from(TokenTree::Token(
DUMMY_SP,
token::Literal(
token::Lit::Str_(Name::intern(&format!(
"assertion failed: {}",
pprust::expr_to_string(&cond_expr).escape_debug()
))),
None,
),
Token::lit(token::Str, Symbol::intern(&format!(
"assertion failed: {}",
pprust::expr_to_string(&cond_expr).escape_debug()
)), None),
))
}).into(),
delim: MacDelimiter::Parenthesis,
......@@ -106,7 +103,7 @@ fn parse_assert<'a>(
//
// Parse this as an actual message, and suggest inserting a comma. Eventually, this should be
// turned into an error.
let custom_message = if let token::Literal(token::Lit::Str_(_), _) = parser.token {
let custom_message = if let token::Literal(token::Lit { kind: token::Str, .. }) = parser.token {
let mut err = cx.struct_span_warn(parser.span, "unexpected string literal");
let comma_span = cx.source_map().next_point(parser.prev_span);
err.span_suggestion_short(
......
......@@ -150,7 +150,7 @@ fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mu
stack.push(tt!(Ident::new(ident.name, false)));
tt!(Punct::new('\'', true))
}
Literal(lit, suffix) => tt!(Literal { lit, suffix }),
Literal(lit) => tt!(Literal { lit }),
DocComment(c) => {
let style = comments::doc_comment_style(&c.as_str());
let stripped = comments::strip_doc_comment_decoration(&c.as_str());
......@@ -161,7 +161,7 @@ fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mu
let stream = vec![
Ident(ast::Ident::new(Symbol::intern("doc"), span), false),
Eq,
Literal(Lit::Str_(Symbol::intern(&escaped)), None),
Token::lit(token::Str, Symbol::intern(&escaped), None),
]
.into_iter()
.map(|token| tokenstream::TokenTree::Token(span, token))
......@@ -215,31 +215,29 @@ fn to_internal(self) -> TokenStream {
return tokenstream::TokenTree::Token(span, token).into();
}
TokenTree::Literal(self::Literal {
lit: Lit::Integer(ref a),
suffix,
lit: token::Lit { kind: token::Integer, symbol, suffix },
span,
}) if a.as_str().starts_with("-") => {
}) if symbol.as_str().starts_with("-") => {
let minus = BinOp(BinOpToken::Minus);
let integer = Symbol::intern(&a.as_str()[1..]);
let integer = Literal(Lit::Integer(integer), suffix);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let integer = Token::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::Token(span, minus);
let b = tokenstream::TokenTree::Token(span, integer);
return vec![a, b].into_iter().collect();
}
TokenTree::Literal(self::Literal {
lit: Lit::Float(ref a),
suffix,
lit: token::Lit { kind: token::Float, symbol, suffix },
span,
}) if a.as_str().starts_with("-") => {
}) if symbol.as_str().starts_with("-") => {
let minus = BinOp(BinOpToken::Minus);
let float = Symbol::intern(&a.as_str()[1..]);
let float = Literal(Lit::Float(float), suffix);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let float = Token::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::Token(span, minus);
let b = tokenstream::TokenTree::Token(span, float);
return vec![a, b].into_iter().collect();
}
TokenTree::Literal(self::Literal { lit, suffix, span }) => {
return tokenstream::TokenTree::Token(span, Literal(lit, suffix)).into()
TokenTree::Literal(self::Literal { lit, span }) => {
return tokenstream::TokenTree::Token(span, Literal(lit)).into()
}
};
......@@ -355,7 +353,6 @@ fn dollar_crate(span: Span) -> Ident {
#[derive(Clone, Debug)]
pub struct Literal {
lit: token::Lit,
suffix: Option<Symbol>,
span: Span,
}
......@@ -381,6 +378,13 @@ pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
call_site: to_span(Transparency::Transparent),
}
}
pub fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal {
Literal {
lit: token::Lit::new(kind, symbol, suffix),
span: server::Span::call_site(self),
}
}
}
impl server::Types for Rustc<'_> {
......@@ -536,59 +540,31 @@ fn debug(&mut self, literal: &Self::Literal) -> String {
format!("{:?}", literal)
}
fn integer(&mut self, n: &str) -> Self::Literal {
Literal {
lit: token::Lit::Integer(Symbol::intern(n)),
suffix: None,
span: server::Span::call_site(self),
}
self.lit(token::Integer, Symbol::intern(n), None)
}
fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
Literal {
lit: token::Lit::Integer(Symbol::intern(n)),
suffix: Some(Symbol::intern(kind)),
span: server::Span::call_site(self),
}
self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind)))
}
fn float(&mut self, n: &str) -> Self::Literal {
Literal {
lit: token::Lit::Float(Symbol::intern(n)),
suffix: None,
span: server::Span::call_site(self),
}
self.lit(token::Float, Symbol::intern(n), None)
}
fn f32(&mut self, n: &str) -> Self::Literal {
Literal {
lit: token::Lit::Float(Symbol::intern(n)),
suffix: Some(Symbol::intern("f32")),
span: server::Span::call_site(self),
}
self.lit(token::Float, Symbol::intern(n), Some(Symbol::intern("f32")))
}
fn f64(&mut self, n: &str) -> Self::Literal {
Literal {
lit: token::Lit::Float(Symbol::intern(n)),
suffix: Some(Symbol::intern("f64")),
span: server::Span::call_site(self),
}
self.lit(token::Float, Symbol::intern(n), Some(Symbol::intern("f64")))
}
fn string(&mut self, string: &str) -> Self::Literal {
let mut escaped = String::new();
for ch in string.chars() {
escaped.extend(ch.escape_debug());
}
Literal {
lit: token::Lit::Str_(Symbol::intern(&escaped)),
suffix: None,
span: server::Span::call_site(self),
}
self.lit(token::Str, Symbol::intern(&escaped), None)
}
fn character(&mut self, ch: char) -> Self::Literal {
let mut escaped = String::new();
escaped.extend(ch.escape_unicode());
Literal {
lit: token::Lit::Char(Symbol::intern(&escaped)),
suffix: None,
span: server::Span::call_site(self),
}
self.lit(token::Char, Symbol::intern(&escaped), None)
}
fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
let string = bytes
......@@ -597,11 +573,7 @@ fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
.flat_map(ascii::escape_default)
.map(Into::<char>::into)
.collect::<String>();
Literal {
lit: token::Lit::ByteStr(Symbol::intern(&string)),
suffix: None,
span: server::Span::call_site(self),
}
self.lit(token::ByteStr, Symbol::intern(&string), None)
}
fn span(&mut self, literal: &Self::Literal) -> Self::Span {
literal.span
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册