Skip to content

Commit 82bb41b

Browse files
alexcrichtonalexcrichton
committed
Merge branch 'master' of https://github.com/Lymia/rust into rollup
2 parents d58abe7 + 57f9c4d commit 82bb41b

40 files changed

+519
-145
lines changed

‎src/libproc_macro/lib.rs

+9-2
Original file line numberDiff line numberDiff line change
@@ -681,7 +681,8 @@ impl TokenTree {
681681
Dollar => op!('$'),
682682
Question => op!('?'),
683683

684-
Ident(ident) | Lifetime(ident) => TokenNode::Term(Term(ident.name)),
684+
Ident(ident, false) | Lifetime(ident) => TokenNode::Term(Term(ident.name)),
685+
Ident(ident, true) => TokenNode::Term(Term(Symbol::intern(&format!("r#{}", ident)))),
685686
Literal(..) | DocComment(..) => TokenNode::Literal(self::Literal(token)),
686687

687688
Interpolated(_) => {
@@ -713,8 +714,14 @@ impl TokenTree {
713714
},
714715
TokenNode::Term(symbol) => {
715716
let ident = ast::Ident { name: symbol.0, ctxt: self.span.0.ctxt() };
717+
let sym_str = symbol.0.as_str();
716718
let token =
717-
if symbol.0.as_str().starts_with("'") { Lifetime(ident) } else { Ident(ident) };
719+
if sym_str.starts_with("'") { Lifetime(ident) }
720+
else if sym_str.starts_with("r#") {
721+
let name = Symbol::intern(&sym_str[2..]);
722+
let ident = ast::Ident { name, ctxt: self.span.0.ctxt() };
723+
Ident(ident, true)
724+
} else { Ident(ident, false) };
718725
return TokenTree::Token(self.span.0, token).into();
719726
}
720727
TokenNode::Literal(token) => return TokenTree::Token(self.span.0, token.0).into(),

‎src/librustc/ich/impls_syntax.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,10 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
318318
opt_name.hash_stable(hcx, hasher);
319319
}
320320

321-
token::Token::Ident(ident) |
321+
token::Token::Ident(ident, is_raw) => {
322+
ident.name.hash_stable(hcx, hasher);
323+
is_raw.hash_stable(hcx, hasher);
324+
}
322325
token::Token::Lifetime(ident) => ident.name.hash_stable(hcx, hasher),
323326

324327
token::Token::Interpolated(_) => {

‎src/librustc_passes/ast_validation.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,13 @@ impl<'a> AstValidator<'a> {
4141
keywords::StaticLifetime.name(),
4242
keywords::Invalid.name()];
4343
if !valid_names.contains(&lifetime.ident.name) &&
44-
token::Ident(lifetime.ident.without_first_quote()).is_reserved_ident() {
44+
token::is_reserved_ident(lifetime.ident.without_first_quote()) {
4545
self.err_handler().span_err(lifetime.span, "lifetimes cannot use keyword names");
4646
}
4747
}
4848

4949
fn check_label(&self, label: Ident, span: Span) {
50-
if token::Ident(label.without_first_quote()).is_reserved_ident() {
50+
if token::is_reserved_ident(label.without_first_quote()) {
5151
self.err_handler().span_err(span, &format!("invalid label name `{}`", label.name));
5252
}
5353
}

‎src/librustc_resolve/lib.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -3206,7 +3206,7 @@ impl<'a> Resolver<'a> {
32063206
// `$crate::a::b`
32073207
module = Some(self.resolve_crate_root(ident.node.ctxt, true));
32083208
continue
3209-
} else if i == 1 && !token::Ident(ident.node).is_path_segment_keyword() {
3209+
} else if i == 1 && !token::is_path_segment_keyword(ident.node) {
32103210
let prev_name = path[0].node.name;
32113211
if prev_name == keywords::Extern.name() ||
32123212
prev_name == keywords::CrateRoot.name() &&

‎src/librustc_resolve/macros.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ impl<'a> base::Resolver for Resolver<'a> {
268268
if k > 0 {
269269
tokens.push(TokenTree::Token(path.span, Token::ModSep).into());
270270
}
271-
let tok = Token::Ident(segment.identifier);
271+
let tok = Token::from_ast_ident(segment.identifier);
272272
tokens.push(TokenTree::Token(path.span, tok).into());
273273
}
274274
}

‎src/librustc_resolve/resolve_imports.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -625,7 +625,7 @@ impl<'a, 'b:'a> ImportResolver<'a, 'b> {
625625
} else {
626626
Some(self.resolve_crate_root(source.ctxt.modern(), false))
627627
}
628-
} else if is_extern && !token::Ident(source).is_path_segment_keyword() {
628+
} else if is_extern && !token::is_path_segment_keyword(source) {
629629
let crate_id =
630630
self.crate_loader.resolve_crate_from_path(source.name, directive.span);
631631
let crate_root =
@@ -667,7 +667,7 @@ impl<'a, 'b:'a> ImportResolver<'a, 'b> {
667667
}
668668
PathResult::Failed(span, msg, true) => {
669669
let (mut self_path, mut self_result) = (module_path.clone(), None);
670-
let is_special = |ident| token::Ident(ident).is_path_segment_keyword() &&
670+
let is_special = |ident| token::is_path_segment_keyword(ident) &&
671671
ident.name != keywords::CrateRoot.name();
672672
if !self_path.is_empty() && !is_special(self_path[0].node) &&
673673
!(self_path.len() > 1 && is_special(self_path[1].node)) {

‎src/librustdoc/html/highlight.rs

+4-4
Original file line numberDiff line numberDiff line change
@@ -323,12 +323,12 @@ impl<'a> Classifier<'a> {
323323
}
324324

325325
// Keywords are also included in the identifier set.
326-
token::Ident(ident) => {
326+
token::Ident(ident, is_raw) => {
327327
match &*ident.name.as_str() {
328-
"ref" | "mut" => Class::RefKeyWord,
328+
"ref" | "mut" if !is_raw => Class::RefKeyWord,
329329

330-
"self" |"Self" => Class::Self_,
331-
"false" | "true" => Class::Bool,
330+
"self" | "Self" => Class::Self_,
331+
"false" | "true" if !is_raw => Class::Bool,
332332

333333
"Option" | "Result" => Class::PreludeTy,
334334
"Some" | "None" | "Ok" | "Err" => Class::PreludeVal,

‎src/libsyntax/ast.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ impl Path {
112112
// or starts with something like `self`/`super`/`$crate`/etc.
113113
pub fn make_root(&self) -> Option<PathSegment> {
114114
if let Some(ident) = self.segments.get(0).map(|seg| seg.identifier) {
115-
if ::parse::token::Ident(ident).is_path_segment_keyword() &&
115+
if ::parse::token::is_path_segment_keyword(ident) &&
116116
ident.name != keywords::Crate.name() {
117117
return None;
118118
}

‎src/libsyntax/attr.rs

+7-6
Original file line numberDiff line numberDiff line change
@@ -1106,17 +1106,18 @@ impl IntType {
11061106

11071107
impl MetaItem {
11081108
fn tokens(&self) -> TokenStream {
1109-
let ident = TokenTree::Token(self.span, Token::Ident(Ident::with_empty_ctxt(self.name)));
1109+
let ident = TokenTree::Token(self.span,
1110+
Token::from_ast_ident(Ident::with_empty_ctxt(self.name)));
11101111
TokenStream::concat(vec![ident.into(), self.node.tokens(self.span)])
11111112
}
11121113

11131114
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
11141115
where I: Iterator<Item = TokenTree>,
11151116
{
11161117
let (span, name) = match tokens.next() {
1117-
Some(TokenTree::Token(span, Token::Ident(ident))) => (span, ident.name),
1118+
Some(TokenTree::Token(span, Token::Ident(ident, _))) => (span, ident.name),
11181119
Some(TokenTree::Token(_, Token::Interpolated(ref nt))) => match nt.0 {
1119-
token::Nonterminal::NtIdent(ident) => (ident.span, ident.node.name),
1120+
token::Nonterminal::NtIdent(ident, _) => (ident.span, ident.node.name),
11201121
token::Nonterminal::NtMeta(ref meta) => return Some(meta.clone()),
11211122
_ => return None,
11221123
},
@@ -1269,14 +1270,14 @@ impl LitKind {
12691270
"true"
12701271
} else {
12711272
"false"
1272-
}))),
1273+
})), false),
12731274
}
12741275
}
12751276

12761277
fn from_token(token: Token) -> Option<LitKind> {
12771278
match token {
1278-
Token::Ident(ident) if ident.name == "true" => Some(LitKind::Bool(true)),
1279-
Token::Ident(ident) if ident.name == "false" => Some(LitKind::Bool(false)),
1279+
Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)),
1280+
Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)),
12801281
Token::Interpolated(ref nt) => match nt.0 {
12811282
token::NtExpr(ref v) => match v.node {
12821283
ExprKind::Lit(ref lit) => Some(lit.node.clone()),

‎src/libsyntax/diagnostics/plugin.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt,
4444
token_tree: &[TokenTree])
4545
-> Box<MacResult+'cx> {
4646
let code = match (token_tree.len(), token_tree.get(0)) {
47-
(1, Some(&TokenTree::Token(_, token::Ident(code)))) => code,
47+
(1, Some(&TokenTree::Token(_, token::Ident(code, _)))) => code,
4848
_ => unreachable!()
4949
};
5050

@@ -82,10 +82,10 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
8282
token_tree.get(1),
8383
token_tree.get(2)
8484
) {
85-
(1, Some(&TokenTree::Token(_, token::Ident(ref code))), None, None) => {
85+
(1, Some(&TokenTree::Token(_, token::Ident(ref code, _))), None, None) => {
8686
(code, None)
8787
},
88-
(3, Some(&TokenTree::Token(_, token::Ident(ref code))),
88+
(3, Some(&TokenTree::Token(_, token::Ident(ref code, _))),
8989
Some(&TokenTree::Token(_, token::Comma)),
9090
Some(&TokenTree::Token(_, token::Literal(token::StrRaw(description, _), None)))) => {
9191
(code, Some(description))
@@ -150,9 +150,9 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt,
150150
let (crate_name, name) = match (&token_tree[0], &token_tree[2]) {
151151
(
152152
// Crate name.
153-
&TokenTree::Token(_, token::Ident(ref crate_name)),
153+
&TokenTree::Token(_, token::Ident(ref crate_name, _)),
154154
// DIAGNOSTICS ident.
155-
&TokenTree::Token(_, token::Ident(ref name))
155+
&TokenTree::Token(_, token::Ident(ref name, _))
156156
) => (*&crate_name, name),
157157
_ => unreachable!()
158158
};

‎src/libsyntax/ext/base.rs

+3-2
Original file line numberDiff line numberDiff line change
@@ -229,8 +229,9 @@ impl<F> TTMacroExpander for F
229229
impl Folder for AvoidInterpolatedIdents {
230230
fn fold_tt(&mut self, tt: tokenstream::TokenTree) -> tokenstream::TokenTree {
231231
if let tokenstream::TokenTree::Token(_, token::Interpolated(ref nt)) = tt {
232-
if let token::NtIdent(ident) = nt.0 {
233-
return tokenstream::TokenTree::Token(ident.span, token::Ident(ident.node));
232+
if let token::NtIdent(ident, is_raw) = nt.0 {
233+
return tokenstream::TokenTree::Token(ident.span,
234+
token::Ident(ident.node, is_raw));
234235
}
235236
}
236237
fold::noop_fold_tt(tt, self)

‎src/libsyntax/ext/quote.rs

+7-5
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ pub mod rt {
7575

7676
impl ToTokens for ast::Ident {
7777
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
78-
vec![TokenTree::Token(DUMMY_SP, token::Ident(*self))]
78+
vec![TokenTree::Token(DUMMY_SP, Token::from_ast_ident(*self))]
7979
}
8080
}
8181

@@ -238,7 +238,9 @@ pub mod rt {
238238
if i > 0 {
239239
inner.push(TokenTree::Token(self.span, token::Colon).into());
240240
}
241-
inner.push(TokenTree::Token(self.span, token::Ident(segment.identifier)).into());
241+
inner.push(TokenTree::Token(
242+
self.span, token::Token::from_ast_ident(segment.identifier)
243+
).into());
242244
}
243245
inner.push(self.tokens.clone());
244246

@@ -658,10 +660,10 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
658660
token::Literal(token::ByteStr(i), suf) => return mk_lit!("ByteStr", suf, i),
659661
token::Literal(token::ByteStrRaw(i, n), suf) => return mk_lit!("ByteStrRaw", suf, i, n),
660662

661-
token::Ident(ident) => {
663+
token::Ident(ident, is_raw) => {
662664
return cx.expr_call(sp,
663665
mk_token_path(cx, sp, "Ident"),
664-
vec![mk_ident(cx, sp, ident)]);
666+
vec![mk_ident(cx, sp, ident), cx.expr_bool(sp, is_raw)]);
665667
}
666668

667669
token::Lifetime(ident) => {
@@ -720,7 +722,7 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
720722

721723
fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt> {
722724
match *tt {
723-
TokenTree::Token(sp, token::Ident(ident)) if quoted => {
725+
TokenTree::Token(sp, token::Ident(ident, _)) if quoted => {
724726
// tt.extend($ident.to_tokens(ext_cx))
725727

726728
let e_to_toks =

‎src/libsyntax/ext/tt/macro_parser.rs

+11-10
Original file line numberDiff line numberDiff line change
@@ -364,8 +364,8 @@ pub fn parse_failure_msg(tok: Token) -> String {
364364

365365
/// Perform a token equality check, ignoring syntax context (that is, an unhygienic comparison)
366366
fn token_name_eq(t1: &Token, t2: &Token) -> bool {
367-
if let (Some(id1), Some(id2)) = (t1.ident(), t2.ident()) {
368-
id1.name == id2.name
367+
if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) {
368+
id1.name == id2.name && is_raw1 == is_raw2
369369
} else if let (&token::Lifetime(id1), &token::Lifetime(id2)) = (t1, t2) {
370370
id1.name == id2.name
371371
} else {
@@ -711,9 +711,10 @@ pub fn parse(
711711

712712
/// The token is an identifier, but not `_`.
713713
/// We prohibit passing `_` to macros expecting `ident` for now.
714-
fn get_macro_ident(token: &Token) -> Option<Ident> {
714+
fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
715715
match *token {
716-
token::Ident(ident) if ident.name != keywords::Underscore.name() => Some(ident),
716+
token::Ident(ident, is_raw) if ident.name != keywords::Underscore.name() =>
717+
Some((ident, is_raw)),
717718
_ => None,
718719
}
719720
}
@@ -737,7 +738,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
737738
"ident" => get_macro_ident(token).is_some(),
738739
"vis" => match *token {
739740
// The follow-set of :vis + "priv" keyword + interpolated
740-
Token::Comma | Token::Ident(_) | Token::Interpolated(_) => true,
741+
Token::Comma | Token::Ident(..) | Token::Interpolated(_) => true,
741742
_ => token.can_begin_type(),
742743
},
743744
"block" => match *token {
@@ -746,7 +747,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
746747
token::NtItem(_)
747748
| token::NtPat(_)
748749
| token::NtTy(_)
749-
| token::NtIdent(_)
750+
| token::NtIdent(..)
750751
| token::NtMeta(_)
751752
| token::NtPath(_)
752753
| token::NtVis(_) => false, // none of these may start with '{'.
@@ -755,15 +756,15 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
755756
_ => false,
756757
},
757758
"path" | "meta" => match *token {
758-
Token::ModSep | Token::Ident(_) => true,
759+
Token::ModSep | Token::Ident(..) => true,
759760
Token::Interpolated(ref nt) => match nt.0 {
760761
token::NtPath(_) | token::NtMeta(_) => true,
761762
_ => may_be_ident(&nt.0),
762763
},
763764
_ => false,
764765
},
765766
"pat" => match *token {
766-
Token::Ident(_) | // box, ref, mut, and other identifiers (can stricten)
767+
Token::Ident(..) | // box, ref, mut, and other identifiers (can stricten)
767768
Token::OpenDelim(token::Paren) | // tuple pattern
768769
Token::OpenDelim(token::Bracket) | // slice pattern
769770
Token::BinOp(token::And) | // reference
@@ -823,9 +824,9 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
823824
"expr" => token::NtExpr(panictry!(p.parse_expr())),
824825
"ty" => token::NtTy(panictry!(p.parse_ty())),
825826
// this could be handled like a token, since it is one
826-
"ident" => if let Some(ident) = get_macro_ident(&p.token) {
827+
"ident" => if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
827828
p.bump();
828-
token::NtIdent(respan(p.prev_span, ident))
829+
token::NtIdent(respan(p.prev_span, ident), is_raw)
829830
} else {
830831
let token_str = pprust::token_to_string(&p.token);
831832
p.fatal(&format!("expected ident, found {}", &token_str)).emit();

‎src/libsyntax/ext/tt/macro_rules.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -831,7 +831,7 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> Result<bool, (String, &'
831831
"pat" => match *tok {
832832
TokenTree::Token(_, ref tok) => match *tok {
833833
FatArrow | Comma | Eq | BinOp(token::Or) => Ok(true),
834-
Ident(i) if i.name == "if" || i.name == "in" => Ok(true),
834+
Ident(i, false) if i.name == "if" || i.name == "in" => Ok(true),
835835
_ => Ok(false)
836836
},
837837
_ => Ok(false),
@@ -840,7 +840,7 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> Result<bool, (String, &'
840840
TokenTree::Token(_, ref tok) => match *tok {
841841
OpenDelim(token::DelimToken::Brace) | OpenDelim(token::DelimToken::Bracket) |
842842
Comma | FatArrow | Colon | Eq | Gt | Semi | BinOp(token::Or) => Ok(true),
843-
Ident(i) if i.name == "as" || i.name == "where" => Ok(true),
843+
Ident(i, false) if i.name == "as" || i.name == "where" => Ok(true),
844844
_ => Ok(false)
845845
},
846846
TokenTree::MetaVarDecl(_, _, frag) if frag.name == "block" => Ok(true),
@@ -860,7 +860,7 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> Result<bool, (String, &'
860860
match *tok {
861861
TokenTree::Token(_, ref tok) => match *tok {
862862
Comma => Ok(true),
863-
Ident(i) if i.name != "priv" => Ok(true),
863+
Ident(i, is_raw) if is_raw || i.name != "priv" => Ok(true),
864864
ref tok => Ok(tok.can_begin_type())
865865
},
866866
TokenTree::MetaVarDecl(_, _, frag) if frag.name == "ident"

‎src/libsyntax/ext/tt/quoted.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ pub fn parse(
200200
let span = match trees.next() {
201201
Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() {
202202
Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() {
203-
Some(kind) => {
203+
Some((kind, _)) => {
204204
let span = end_sp.with_lo(start_sp.lo());
205205
result.push(TokenTree::MetaVarDecl(span, ident, kind));
206206
continue;
@@ -289,14 +289,14 @@ where
289289
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
290290
// metavariable that names the crate of the invokation.
291291
Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => {
292-
let ident = token.ident().unwrap();
292+
let (ident, _) = token.ident().unwrap();
293293
let span = ident_span.with_lo(span.lo());
294294
if ident.name == keywords::Crate.name() {
295295
let ident = ast::Ident {
296296
name: keywords::DollarCrate.name(),
297297
..ident
298298
};
299-
TokenTree::Token(span, token::Ident(ident))
299+
TokenTree::Token(span, token::Ident(ident, false))
300300
} else {
301301
TokenTree::MetaVar(span, ident)
302302
}

‎src/libsyntax/ext/tt/transcribe.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ pub fn transcribe(cx: &ExtCtxt,
169169
Ident { ctxt: ident.ctxt.apply_mark(cx.current_expansion.mark), ..ident };
170170
sp = sp.with_ctxt(sp.ctxt().apply_mark(cx.current_expansion.mark));
171171
result.push(TokenTree::Token(sp, token::Dollar).into());
172-
result.push(TokenTree::Token(sp, token::Ident(ident)).into());
172+
result.push(TokenTree::Token(sp, token::Token::from_ast_ident(ident)).into());
173173
}
174174
}
175175
quoted::TokenTree::Delimited(mut span, delimited) => {

0 commit comments

Comments
 (0)