Skip to content

Commit 14382c6

Browse files
Aaron1011Aaron1011
committed
Collect tokens for ast::Expr 
1 parent ff991d6 commit 14382c6

File tree

16 files changed

+97
-18
lines changed

16 files changed

+97
-18
lines changed

‎src/librustc_ast/ast.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -1006,11 +1006,12 @@ pub struct Expr {
10061006
pub kind: ExprKind,
10071007
pub span: Span,
10081008
pub attrs: AttrVec,
1009+
pub tokens: Option<TokenStream>,
10091010
}
10101011

10111012
// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
10121013
#[cfg(target_arch = "x86_64")]
1013-
rustc_data_structures::static_assert_size!(Expr, 96);
1014+
rustc_data_structures::static_assert_size!(Expr, 104);
10141015

10151016
impl Expr {
10161017
/// Returns `true` if this expression would be valid somewhere that expects a value;

‎src/librustc_ast/mut_visit.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -1095,7 +1095,10 @@ pub fn noop_visit_anon_const<T: MutVisitor>(AnonConst { id, value }: &mut AnonCo
10951095
vis.visit_expr(value);
10961096
}
10971097

1098-
pub fn noop_visit_expr<T: MutVisitor>(Expr { kind, id, span, attrs }: &mut Expr, vis: &mut T) {
1098+
pub fn noop_visit_expr<T: MutVisitor>(
1099+
Expr { kind, id, span, attrs, tokens: _ }: &mut Expr,
1100+
vis: &mut T,
1101+
) {
10991102
match kind {
11001103
ExprKind::Box(expr) => vis.visit_expr(expr),
11011104
ExprKind::Array(exprs) => visit_exprs(exprs, vis),

‎src/librustc_ast_lowering/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1126,6 +1126,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
11261126
kind: ExprKind::Path(qself.clone(), path.clone()),
11271127
span: ty.span,
11281128
attrs: AttrVec::new(),
1129+
tokens: None,
11291130
};
11301131

11311132
let ct = self.with_new_scopes(|this| hir::AnonConst {

‎src/librustc_builtin_macros/asm.rs

+1
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,7 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast
519519
kind: ast::ExprKind::InlineAsm(inline_asm),
520520
span: sp,
521521
attrs: ast::AttrVec::new(),
522+
tokens: None,
522523
})
523524
}
524525

‎src/librustc_builtin_macros/concat_idents.rs

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ pub fn expand_concat_idents<'cx>(
5252
kind: ast::ExprKind::Path(None, ast::Path::from_ident(self.ident)),
5353
span: self.ident.span,
5454
attrs: ast::AttrVec::new(),
55+
tokens: None,
5556
}))
5657
}
5758

‎src/librustc_builtin_macros/llvm_asm.rs

+1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ pub fn expand_llvm_asm<'cx>(
6161
kind: ast::ExprKind::LlvmInlineAsm(P(inline_asm)),
6262
span: cx.with_def_site_ctxt(sp),
6363
attrs: ast::AttrVec::new(),
64+
tokens: None,
6465
}))
6566
}
6667

‎src/librustc_expand/base.rs

+1
Original file line numberDiff line numberDiff line change
@@ -594,6 +594,7 @@ impl DummyResult {
594594
kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) },
595595
span: sp,
596596
attrs: ast::AttrVec::new(),
597+
tokens: None,
597598
})
598599
}
599600

‎src/librustc_expand/build.rs

+8-2
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,13 @@ impl<'a> ExtCtxt<'a> {
7070
pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst {
7171
ast::AnonConst {
7272
id: ast::DUMMY_NODE_ID,
73-
value: P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }),
73+
value: P(ast::Expr {
74+
id: ast::DUMMY_NODE_ID,
75+
kind,
76+
span,
77+
attrs: AttrVec::new(),
78+
tokens: None,
79+
}),
7480
}
7581
}
7682

@@ -205,7 +211,7 @@ impl<'a> ExtCtxt<'a> {
205211
}
206212

207213
pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P<ast::Expr> {
208-
P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() })
214+
P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new(), tokens: None })
209215
}
210216

211217
pub fn expr_path(&self, path: ast::Path) -> P<ast::Expr> {

‎src/librustc_expand/placeholders.rs

+1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ pub fn placeholder(
3434
span,
3535
attrs: ast::AttrVec::new(),
3636
kind: ast::ExprKind::MacCall(mac_placeholder()),
37+
tokens: None,
3738
})
3839
};
3940
let ty = || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span });

‎src/librustc_interface/util.rs

+2
Original file line numberDiff line numberDiff line change
@@ -713,6 +713,7 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> {
713713
kind: ast::ExprKind::Block(P(b), None),
714714
span: rustc_span::DUMMY_SP,
715715
attrs: AttrVec::new(),
716+
tokens: None,
716717
});
717718

718719
ast::Stmt {
@@ -728,6 +729,7 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> {
728729
id: self.resolver.next_node_id(),
729730
span: rustc_span::DUMMY_SP,
730731
attrs: AttrVec::new(),
732+
tokens: None,
731733
});
732734

733735
let loop_stmt = ast::Stmt {

‎src/librustc_parse/lib.rs

+8
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,12 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
272272
Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())
273273
}
274274
Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
275+
Nonterminal::NtExpr(ref expr) => {
276+
if expr.tokens.is_none() {
277+
debug!("missing tokens for expr {:?}", expr);
278+
}
279+
prepend_attrs(sess, &expr.attrs, expr.tokens.as_ref(), span)
280+
}
275281
_ => None,
276282
};
277283

@@ -311,6 +317,8 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
311317
"cached tokens found, but they're not \"probably equal\", \
312318
going with stringified version"
313319
);
320+
info!("cached tokens: {:?}", tokens);
321+
info!("reparsed tokens: {:?}", tokens_for_real);
314322
}
315323
tokens_for_real
316324
}

‎src/librustc_parse/parser/diagnostics.rs

+1
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ impl RecoverQPath for Expr {
9595
kind: ExprKind::Path(qself, path),
9696
attrs: AttrVec::new(),
9797
id: ast::DUMMY_NODE_ID,
98+
tokens: None,
9899
}
99100
}
100101
}

‎src/librustc_parse/parser/expr.rs

+34-14
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use super::{BlockMode, Parser, PathStyle, Restrictions, TokenType};
44
use super::{SemiColonMode, SeqSep, TokenExpectType};
55
use crate::maybe_recover_from_interpolated_ty_qpath;
66

7+
use log::debug;
78
use rustc_ast::ast::{self, AttrStyle, AttrVec, CaptureBy, Field, Lit, UnOp, DUMMY_NODE_ID};
89
use rustc_ast::ast::{AnonConst, BinOp, BinOpKind, FnDecl, FnRetTy, MacCall, Param, Ty, TyKind};
910
use rustc_ast::ast::{Arm, Async, BlockCheckMode, Expr, ExprKind, Label, Movability, RangeLimits};
@@ -431,19 +432,23 @@ impl<'a> Parser<'a> {
431432
/// Parses a prefix-unary-operator expr.
432433
fn parse_prefix_expr(&mut self, attrs: Option<AttrVec>) -> PResult<'a, P<Expr>> {
433434
let attrs = self.parse_or_use_outer_attributes(attrs)?;
434-
let lo = self.token.span;
435-
// Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
436-
let (hi, ex) = match self.token.uninterpolate().kind {
437-
token::Not => self.parse_unary_expr(lo, UnOp::Not), // `!expr`
438-
token::Tilde => self.recover_tilde_expr(lo), // `~expr`
439-
token::BinOp(token::Minus) => self.parse_unary_expr(lo, UnOp::Neg), // `-expr`
440-
token::BinOp(token::Star) => self.parse_unary_expr(lo, UnOp::Deref), // `*expr`
441-
token::BinOp(token::And) | token::AndAnd => self.parse_borrow_expr(lo),
442-
token::Ident(..) if self.token.is_keyword(kw::Box) => self.parse_box_expr(lo),
443-
token::Ident(..) if self.is_mistaken_not_ident_negation() => self.recover_not_expr(lo),
444-
_ => return self.parse_dot_or_call_expr(Some(attrs)),
445-
}?;
446-
Ok(self.mk_expr(lo.to(hi), ex, attrs))
435+
self.maybe_collect_tokens(!attrs.is_empty(), |this| {
436+
let lo = this.token.span;
437+
// Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
438+
let (hi, ex) = match this.token.uninterpolate().kind {
439+
token::Not => this.parse_unary_expr(lo, UnOp::Not), // `!expr`
440+
token::Tilde => this.recover_tilde_expr(lo), // `~expr`
441+
token::BinOp(token::Minus) => this.parse_unary_expr(lo, UnOp::Neg), // `-expr`
442+
token::BinOp(token::Star) => this.parse_unary_expr(lo, UnOp::Deref), // `*expr`
443+
token::BinOp(token::And) | token::AndAnd => this.parse_borrow_expr(lo),
444+
token::Ident(..) if this.token.is_keyword(kw::Box) => this.parse_box_expr(lo),
445+
token::Ident(..) if this.is_mistaken_not_ident_negation() => {
446+
this.recover_not_expr(lo)
447+
}
448+
_ => return this.parse_dot_or_call_expr(Some(attrs)),
449+
}?;
450+
Ok(this.mk_expr(lo.to(hi), ex, attrs))
451+
})
447452
}
448453

449454
fn parse_prefix_expr_common(&mut self, lo: Span) -> PResult<'a, (Span, P<Expr>)> {
@@ -998,6 +1003,21 @@ impl<'a> Parser<'a> {
9981003
}
9991004
}
10001005

1006+
fn maybe_collect_tokens(
1007+
&mut self,
1008+
has_outer_attrs: bool,
1009+
f: impl FnOnce(&mut Self) -> PResult<'a, P<Expr>>,
1010+
) -> PResult<'a, P<Expr>> {
1011+
if has_outer_attrs {
1012+
let (mut expr, tokens) = self.collect_tokens(f)?;
1013+
debug!("maybe_collect_tokens: Collected tokens for {:?} (tokens {:?}", expr, tokens);
1014+
expr.tokens = Some(tokens);
1015+
Ok(expr)
1016+
} else {
1017+
f(self)
1018+
}
1019+
}
1020+
10011021
fn parse_lit_expr(&mut self, attrs: AttrVec) -> PResult<'a, P<Expr>> {
10021022
let lo = self.token.span;
10031023
match self.parse_opt_lit() {
@@ -2169,7 +2189,7 @@ impl<'a> Parser<'a> {
21692189
}
21702190

21712191
crate fn mk_expr(&self, span: Span, kind: ExprKind, attrs: AttrVec) -> P<Expr> {
2172-
P(Expr { kind, span, attrs, id: DUMMY_NODE_ID })
2192+
P(Expr { kind, span, attrs, id: DUMMY_NODE_ID, tokens: None })
21732193
}
21742194

21752195
pub(super) fn mk_expr_err(&self, span: Span) -> P<Expr> {

‎src/test/ui-fulldeps/pprust-expr-roundtrip.rs

+2
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ fn expr(kind: ExprKind) -> P<Expr> {
5656
kind,
5757
span: DUMMY_SP,
5858
attrs: ThinVec::new(),
59+
tokens: None
5960
})
6061
}
6162

@@ -200,6 +201,7 @@ impl MutVisitor for AddParens {
200201
kind: ExprKind::Paren(e),
201202
span: DUMMY_SP,
202203
attrs: ThinVec::new(),
204+
tokens: None
203205
})
204206
});
205207
}

‎src/test/ui/proc-macro/keep-expr-tokens.rs

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// aux-build:test-macros.rs
2+
3+
#![feature(stmt_expr_attributes)]
4+
#![feature(proc_macro_hygiene)]
5+
6+
extern crate test_macros;
7+
8+
use test_macros::recollect_attr;
9+
10+
fn main() {
11+
#[test_macros::recollect_attr]
12+
for item in missing_fn() {} //~ ERROR cannot find
13+
14+
(#[recollect_attr] #[recollect_attr] ((#[recollect_attr] bad))); //~ ERROR cannot
15+
}

‎src/test/ui/proc-macro/keep-expr-tokens.stderr

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
error[E0425]: cannot find function `missing_fn` in this scope
2+
--> $DIR/keep-expr-tokens.rs:12:17
3+
|
4+
LL | for item in missing_fn() {}
5+
| ^^^^^^^^^^ not found in this scope
6+
7+
error[E0425]: cannot find value `bad` in this scope
8+
--> $DIR/keep-expr-tokens.rs:14:62
9+
|
10+
LL | (#[recollect_attr] #[recollect_attr] ((#[recollect_attr] bad)));
11+
| ^^^ not found in this scope
12+
13+
error: aborting due to 2 previous errors
14+
15+
For more information about this error, try `rustc --explain E0425`.

0 commit comments

Comments
 (0)