Skip to content

Commit a961e67

Browse files
Aaron1011Aaron1011
committed
Set tokens on AST node in collect_tokens
A new `HasTokens` trait is introduced, which is used to move logic from the callers of `collect_tokens` into the body of `collect_tokens`. In addition to reducing duplication, this paves the way for PR #80689, which needs to perform additional logic during token collection.
1 parent 9bc8b00 commit a961e67

File tree

7 files changed

+101
-147
lines changed

7 files changed

+101
-147
lines changed

‎compiler/rustc_ast/src/ast.rs

+66-10
Original file line numberDiff line numberDiff line change
@@ -916,16 +916,6 @@ impl Stmt {
916916
}
917917
}
918918

919-
pub fn set_tokens(&mut self, tokens: Option<LazyTokenStream>) {
920-
match self.kind {
921-
StmtKind::Local(ref mut local) => local.tokens = tokens,
922-
StmtKind::Item(ref mut item) => item.tokens = tokens,
923-
StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens = tokens,
924-
StmtKind::Empty => {}
925-
StmtKind::MacCall(ref mut mac) => mac.tokens = tokens,
926-
}
927-
}
928-
929919
pub fn has_trailing_semicolon(&self) -> bool {
930920
match &self.kind {
931921
StmtKind::Semi(_) => true,
@@ -2881,3 +2871,69 @@ impl TryFrom<ItemKind> for ForeignItemKind {
28812871
}
28822872

28832873
pub type ForeignItem = Item<ForeignItemKind>;
2874+
2875+
pub trait HasTokens {
2876+
/// Called by `Parser::collect_tokens` to store the collected
2877+
/// tokens inside an AST node
2878+
fn finalize_tokens(&mut self, tokens: LazyTokenStream);
2879+
}
2880+
2881+
impl<T: HasTokens + 'static> HasTokens for P<T> {
2882+
fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
2883+
(**self).finalize_tokens(tokens);
2884+
}
2885+
}
2886+
2887+
impl<T: HasTokens> HasTokens for Option<T> {
2888+
fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
2889+
if let Some(inner) = self {
2890+
inner.finalize_tokens(tokens);
2891+
}
2892+
}
2893+
}
2894+
2895+
impl HasTokens for Attribute {
2896+
fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
2897+
match &mut self.kind {
2898+
AttrKind::Normal(_, attr_tokens) => {
2899+
if attr_tokens.is_none() {
2900+
*attr_tokens = Some(tokens);
2901+
}
2902+
}
2903+
AttrKind::DocComment(..) => {
2904+
panic!("Called finalize_tokens on doc comment attr {:?}", self)
2905+
}
2906+
}
2907+
}
2908+
}
2909+
2910+
impl HasTokens for Stmt {
2911+
fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
2912+
let stmt_tokens = match self.kind {
2913+
StmtKind::Local(ref mut local) => &mut local.tokens,
2914+
StmtKind::Item(ref mut item) => &mut item.tokens,
2915+
StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => &mut expr.tokens,
2916+
StmtKind::Empty => return,
2917+
StmtKind::MacCall(ref mut mac) => &mut mac.tokens,
2918+
};
2919+
if stmt_tokens.is_none() {
2920+
*stmt_tokens = Some(tokens);
2921+
}
2922+
}
2923+
}
2924+
2925+
macro_rules! derive_has_tokens {
2926+
($($ty:path),*) => { $(
2927+
impl HasTokens for $ty {
2928+
fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
2929+
if self.tokens.is_none() {
2930+
self.tokens = Some(tokens);
2931+
}
2932+
}
2933+
}
2934+
)* }
2935+
}
2936+
2937+
derive_has_tokens! {
2938+
Item, Expr, Ty, AttrItem, Visibility, Path, Block, Pat
2939+
}

‎compiler/rustc_parse/src/parser/attr.rs

+4-12
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ impl<'a> Parser<'a> {
8989
inner_parse_policy, self.token
9090
);
9191
let lo = self.token.span;
92-
let ((item, style, span), tokens) = self.collect_tokens(|this| {
92+
self.collect_tokens(|this| {
9393
if this.eat(&token::Pound) {
9494
let style = if this.eat(&token::Not) {
9595
ast::AttrStyle::Inner
@@ -107,15 +107,13 @@ impl<'a> Parser<'a> {
107107
this.error_on_forbidden_inner_attr(attr_sp, inner_parse_policy);
108108
}
109109

110-
Ok((item, style, attr_sp))
110+
Ok(attr::mk_attr_from_item(item, None, style, attr_sp))
111111
} else {
112112
let token_str = pprust::token_to_string(&this.token);
113113
let msg = &format!("expected `#`, found `{}`", token_str);
114114
Err(this.struct_span_err(this.token.span, msg))
115115
}
116-
})?;
117-
118-
Ok(attr::mk_attr_from_item(item, tokens, style, span))
116+
})
119117
}
120118

121119
pub(super) fn error_on_forbidden_inner_attr(&self, attr_sp: Span, policy: InnerAttrPolicy<'_>) {
@@ -165,13 +163,7 @@ impl<'a> Parser<'a> {
165163
let args = this.parse_attr_args()?;
166164
Ok(ast::AttrItem { path, args, tokens: None })
167165
};
168-
if capture_tokens {
169-
let (mut item, tokens) = self.collect_tokens(do_parse)?;
170-
item.tokens = tokens;
171-
item
172-
} else {
173-
do_parse(self)?
174-
}
166+
if capture_tokens { self.collect_tokens(do_parse) } else { do_parse(self) }?
175167
})
176168
}
177169

‎compiler/rustc_parse/src/parser/expr.rs

+4-16
Original file line numberDiff line numberDiff line change
@@ -472,7 +472,8 @@ impl<'a> Parser<'a> {
472472
/// Parses a prefix-unary-operator expr.
473473
fn parse_prefix_expr(&mut self, attrs: Option<AttrVec>) -> PResult<'a, P<Expr>> {
474474
let attrs = self.parse_or_use_outer_attributes(attrs)?;
475-
self.maybe_collect_tokens(super::attr::maybe_needs_tokens(&attrs), |this| {
475+
let needs_tokens = super::attr::maybe_needs_tokens(&attrs);
476+
let do_parse = |this: &mut Parser<'a>| {
476477
let lo = this.token.span;
477478
// Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
478479
let (hi, ex) = match this.token.uninterpolate().kind {
@@ -488,7 +489,8 @@ impl<'a> Parser<'a> {
488489
_ => return this.parse_dot_or_call_expr(Some(attrs)),
489490
}?;
490491
Ok(this.mk_expr(lo.to(hi), ex, attrs))
491-
})
492+
};
493+
if needs_tokens { self.collect_tokens(do_parse) } else { do_parse(self) }
492494
}
493495

494496
fn parse_prefix_expr_common(&mut self, lo: Span) -> PResult<'a, (Span, P<Expr>)> {
@@ -1125,20 +1127,6 @@ impl<'a> Parser<'a> {
11251127
}
11261128
}
11271129

1128-
fn maybe_collect_tokens(
1129-
&mut self,
1130-
needs_tokens: bool,
1131-
f: impl FnOnce(&mut Self) -> PResult<'a, P<Expr>>,
1132-
) -> PResult<'a, P<Expr>> {
1133-
if needs_tokens {
1134-
let (mut expr, tokens) = self.collect_tokens(f)?;
1135-
expr.tokens = tokens;
1136-
Ok(expr)
1137-
} else {
1138-
f(self)
1139-
}
1140-
}
1141-
11421130
fn parse_lit_expr(&mut self, attrs: AttrVec) -> PResult<'a, P<Expr>> {
11431131
let lo = self.token.span;
11441132
match self.parse_opt_lit() {

‎compiler/rustc_parse/src/parser/item.rs

+1-13
Original file line numberDiff line numberDiff line change
@@ -125,19 +125,7 @@ impl<'a> Parser<'a> {
125125
item
126126
};
127127

128-
let (mut item, tokens) = if needs_tokens {
129-
let (item, tokens) = self.collect_tokens(parse_item)?;
130-
(item, tokens)
131-
} else {
132-
(parse_item(self)?, None)
133-
};
134-
if let Some(item) = &mut item {
135-
// If we captured tokens during parsing (due to encountering an `NtItem`),
136-
// use those instead
137-
if item.tokens.is_none() {
138-
item.tokens = tokens;
139-
}
140-
}
128+
let item = if needs_tokens { self.collect_tokens(parse_item) } else { parse_item(self) }?;
141129

142130
self.unclosed_delims.append(&mut unclosed_delims);
143131
Ok(item)

‎compiler/rustc_parse/src/parser/mod.rs

+7-6
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ use rustc_ast::token::{self, DelimToken, Token, TokenKind};
1919
use rustc_ast::tokenstream::{self, DelimSpan, LazyTokenStream, Spacing};
2020
use rustc_ast::tokenstream::{CreateTokenStream, TokenStream, TokenTree, TreeAndSpacing};
2121
use rustc_ast::DUMMY_NODE_ID;
22-
use rustc_ast::{self as ast, AnonConst, AttrStyle, AttrVec, Const, CrateSugar, Extern, Unsafe};
23-
use rustc_ast::{Async, Expr, ExprKind, MacArgs, MacDelimiter, Mutability, StrLit};
22+
use rustc_ast::{self as ast, AnonConst, AttrStyle, AttrVec, Const, CrateSugar, Extern, HasTokens};
23+
use rustc_ast::{Async, Expr, ExprKind, MacArgs, MacDelimiter, Mutability, StrLit, Unsafe};
2424
use rustc_ast::{Visibility, VisibilityKind};
2525
use rustc_ast_pretty::pprust;
2626
use rustc_data_structures::sync::Lrc;
@@ -1234,10 +1234,10 @@ impl<'a> Parser<'a> {
12341234
/// This restriction shouldn't be an issue in practice,
12351235
/// since this function is used to record the tokens for
12361236
/// a parsed AST item, which always has matching delimiters.
1237-
pub fn collect_tokens<R>(
1237+
pub fn collect_tokens<R: HasTokens>(
12381238
&mut self,
12391239
f: impl FnOnce(&mut Self) -> PResult<'a, R>,
1240-
) -> PResult<'a, (R, Option<LazyTokenStream>)> {
1240+
) -> PResult<'a, R> {
12411241
let start_token = (self.token.clone(), self.token_spacing);
12421242
let cursor_snapshot = TokenCursor {
12431243
frame: self.token_cursor.frame.clone(),
@@ -1249,7 +1249,7 @@ impl<'a> Parser<'a> {
12491249
append_unglued_token: self.token_cursor.append_unglued_token.clone(),
12501250
};
12511251

1252-
let ret = f(self)?;
1252+
let mut ret = f(self)?;
12531253

12541254
// Produces a `TokenStream` on-demand. Using `cursor_snapshot`
12551255
// and `num_calls`, we can reconstruct the `TokenStream` seen
@@ -1319,7 +1319,8 @@ impl<'a> Parser<'a> {
13191319
trailing_semi: false,
13201320
append_unglued_token: self.token_cursor.append_unglued_token.clone(),
13211321
};
1322-
Ok((ret, Some(LazyTokenStream::new(lazy_impl))))
1322+
ret.finalize_tokens(LazyTokenStream::new(lazy_impl));
1323+
Ok(ret)
13231324
}
13241325

13251326
/// `::{` or `::*`

‎compiler/rustc_parse/src/parser/nonterminal.rs

+18-81
Original file line numberDiff line numberDiff line change
@@ -99,80 +99,34 @@ impl<'a> Parser<'a> {
9999
// we always capture tokens for any `Nonterminal` which needs them.
100100
Ok(match kind {
101101
NonterminalKind::Item => match self.collect_tokens(|this| this.parse_item())? {
102-
(Some(mut item), tokens) => {
103-
// If we captured tokens during parsing (due to outer attributes),
104-
// use those.
105-
if item.tokens.is_none() {
106-
item.tokens = tokens;
107-
}
108-
token::NtItem(item)
109-
}
110-
(None, _) => {
102+
Some(item) => token::NtItem(item),
103+
None => {
111104
return Err(self.struct_span_err(self.token.span, "expected an item keyword"));
112105
}
113106
},
114107
NonterminalKind::Block => {
115-
let (mut block, tokens) = self.collect_tokens(|this| this.parse_block())?;
116-
// We have have eaten an NtBlock, which could already have tokens
117-
if block.tokens.is_none() {
118-
block.tokens = tokens;
119-
}
120-
token::NtBlock(block)
108+
token::NtBlock(self.collect_tokens(|this| this.parse_block())?)
121109
}
122-
NonterminalKind::Stmt => {
123-
let (stmt, tokens) = self.collect_tokens(|this| this.parse_stmt())?;
124-
match stmt {
125-
Some(mut s) => {
126-
if s.tokens().is_none() {
127-
s.set_tokens(tokens);
128-
}
129-
token::NtStmt(s)
130-
}
131-
None => {
132-
return Err(self.struct_span_err(self.token.span, "expected a statement"));
133-
}
110+
NonterminalKind::Stmt => match self.collect_tokens(|this| this.parse_stmt())? {
111+
Some(s) => token::NtStmt(s),
112+
None => {
113+
return Err(self.struct_span_err(self.token.span, "expected a statement"));
134114
}
135-
}
115+
},
136116
NonterminalKind::Pat2018 { .. } | NonterminalKind::Pat2021 { .. } => {
137-
let (mut pat, tokens) = self.collect_tokens(|this| match kind {
117+
token::NtPat(self.collect_tokens(|this| match kind {
138118
NonterminalKind::Pat2018 { .. } => this.parse_pat(None),
139119
NonterminalKind::Pat2021 { .. } => {
140120
this.parse_top_pat(GateOr::Yes, RecoverComma::No)
141121
}
142122
_ => unreachable!(),
143-
})?;
144-
// We have have eaten an NtPat, which could already have tokens
145-
if pat.tokens.is_none() {
146-
pat.tokens = tokens;
147-
}
148-
token::NtPat(pat)
149-
}
150-
NonterminalKind::Expr => {
151-
let (mut expr, tokens) = self.collect_tokens(|this| this.parse_expr())?;
152-
// If we captured tokens during parsing (due to outer attributes),
153-
// use those.
154-
if expr.tokens.is_none() {
155-
expr.tokens = tokens;
156-
}
157-
token::NtExpr(expr)
123+
})?)
158124
}
125+
NonterminalKind::Expr => token::NtExpr(self.collect_tokens(|this| this.parse_expr())?),
159126
NonterminalKind::Literal => {
160-
let (mut lit, tokens) =
161-
self.collect_tokens(|this| this.parse_literal_maybe_minus())?;
162-
// We have have eaten a nonterminal, which could already have tokens
163-
if lit.tokens.is_none() {
164-
lit.tokens = tokens;
165-
}
166-
token::NtLiteral(lit)
167-
}
168-
NonterminalKind::Ty => {
169-
let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?;
170-
// We have an eaten an NtTy, which could already have tokens
171-
if ty.tokens.is_none() {
172-
ty.tokens = tokens;
173-
}
174-
token::NtTy(ty)
127+
token::NtLiteral(self.collect_tokens(|this| this.parse_literal_maybe_minus())?)
175128
}
129+
NonterminalKind::Ty => token::NtTy(self.collect_tokens(|this| this.parse_ty())?),
176130
// this could be handled like a token, since it is one
177131
NonterminalKind::Ident => {
178132
if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
@@ -185,32 +139,15 @@ impl<'a> Parser<'a> {
185139
}
186140
}
187141
NonterminalKind::Path => {
188-
let (mut path, tokens) =
189-
self.collect_tokens(|this| this.parse_path(PathStyle::Type))?;
190-
// We have have eaten an NtPath, which could already have tokens
191-
if path.tokens.is_none() {
192-
path.tokens = tokens;
193-
}
194-
token::NtPath(path)
142+
token::NtPath(self.collect_tokens(|this| this.parse_path(PathStyle::Type))?)
195143
}
196144
NonterminalKind::Meta => {
197-
let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item(false))?;
198-
// We may have eaten a nonterminal, which could already have tokens
199-
if attr.tokens.is_none() {
200-
attr.tokens = tokens;
201-
}
202-
token::NtMeta(P(attr))
145+
token::NtMeta(P(self.collect_tokens(|this| this.parse_attr_item(false))?))
203146
}
204147
NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
205-
NonterminalKind::Vis => {
206-
let (mut vis, tokens) =
207-
self.collect_tokens(|this| this.parse_visibility(FollowedByType::Yes))?;
208-
// We may have etan an `NtVis`, which could already have tokens
209-
if vis.tokens.is_none() {
210-
vis.tokens = tokens;
211-
}
212-
token::NtVis(vis)
213-
}
148+
NonterminalKind::Vis => token::NtVis(
149+
self.collect_tokens(|this| this.parse_visibility(FollowedByType::Yes))?,
150+
),
214151
NonterminalKind::Lifetime => {
215152
if self.check_lifetime() {
216153
token::NtLifetime(self.expect_lifetime().ident)

‎compiler/rustc_parse/src/parser/stmt.rs

+1-9
Original file line numberDiff line numberDiff line change
@@ -89,15 +89,7 @@ impl<'a> Parser<'a> {
8989
};
9090

9191
let stmt = if has_attrs {
92-
let (mut stmt, tokens) = self.collect_tokens(parse_stmt_inner)?;
93-
if let Some(stmt) = &mut stmt {
94-
// If we already have tokens (e.g. due to encounting an `NtStmt`),
95-
// use those instead.
96-
if stmt.tokens().is_none() {
97-
stmt.set_tokens(tokens);
98-
}
99-
}
100-
stmt
92+
self.collect_tokens(parse_stmt_inner)?
10193
} else {
10294
parse_stmt_inner(self)?
10395
};

0 commit comments

Comments
 (0)