Skip to content

Commit 1d79660

Browse files
nnethercotennethercote
committed
Remove NtItem and NtStmt.
This involves replacing `nt_pretty_printing_compatibility_hack` with `stream_pretty_printing_compatibility_hack`. The handling of statements in `transcribe` is slightly different to other nonterminal kinds, due to the lack of `from_ast` implementation for empty statements. Notable test changes: - `tests/ui/macros/nonterminal-matching.rs` now passes. Removal of `Token::Interpolated` will remove the "captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens" restriction. - `tests/ui/proc-macro/expand-to-derive.rs`: the diff looks large but the only difference is the insertion of a single invisible-delimited group. I hope this doesn't cause any new `ProceduralMasquerade`-type problems.
1 parent 4574033 commit 1d79660

23 files changed

+251
-305
lines changed

‎compiler/rustc_ast/src/ast_traits.rs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -231,8 +231,6 @@ impl HasTokens for Attribute {
231231
impl HasTokens for Nonterminal {
232232
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
233233
match self {
234-
Nonterminal::NtItem(item) => item.tokens(),
235-
Nonterminal::NtStmt(stmt) => stmt.tokens(),
236234
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
237235
Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
238236
Nonterminal::NtPath(path) => path.tokens(),
@@ -241,8 +239,6 @@ impl HasTokens for Nonterminal {
241239
}
242240
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
243241
match self {
244-
Nonterminal::NtItem(item) => item.tokens_mut(),
245-
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
246242
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
247243
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
248244
Nonterminal::NtPath(path) => path.tokens_mut(),

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -822,19 +822,7 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
822822
// multiple items there....
823823
fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
824824
match nt {
825-
token::NtItem(item) => visit_clobber(item, |item| {
826-
// This is probably okay, because the only visitors likely to
827-
// peek inside interpolated nodes will be renamings/markings,
828-
// which map single items to single items.
829-
vis.flat_map_item(item).expect_one("expected visitor to produce exactly one item")
830-
}),
831825
token::NtBlock(block) => vis.visit_block(block),
832-
token::NtStmt(stmt) => visit_clobber(stmt, |stmt| {
833-
// See reasoning above.
834-
stmt.map(|stmt| {
835-
vis.flat_map_stmt(stmt).expect_one("expected visitor to produce exactly one item")
836-
})
837-
}),
838826
token::NtExpr(expr) => vis.visit_expr(expr),
839827
token::NtLiteral(expr) => vis.visit_expr(expr),
840828
token::NtMeta(item) => {

‎compiler/rustc_ast/src/token.rs

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -918,9 +918,7 @@ impl PartialEq<TokenKind> for Token {
918918
#[derive(Clone, Encodable, Decodable)]
919919
/// For interpolation during macro expansion.
920920
pub enum Nonterminal {
921-
NtItem(P<ast::Item>),
922921
NtBlock(P<ast::Block>),
923-
NtStmt(P<ast::Stmt>),
924922
NtExpr(P<ast::Expr>),
925923
NtLiteral(P<ast::Expr>),
926924
/// Stuff inside brackets for attributes
@@ -1009,9 +1007,7 @@ impl fmt::Display for NonterminalKind {
10091007
impl Nonterminal {
10101008
pub fn use_span(&self) -> Span {
10111009
match self {
1012-
NtItem(item) => item.span,
10131010
NtBlock(block) => block.span,
1014-
NtStmt(stmt) => stmt.span,
10151011
NtExpr(expr) | NtLiteral(expr) => expr.span,
10161012
NtMeta(attr_item) => attr_item.span(),
10171013
NtPath(path) => path.span,
@@ -1020,9 +1016,7 @@ impl Nonterminal {
10201016

10211017
pub fn descr(&self) -> &'static str {
10221018
match self {
1023-
NtItem(..) => "item",
10241019
NtBlock(..) => "block",
1025-
NtStmt(..) => "statement",
10261020
NtExpr(..) => "expression",
10271021
NtLiteral(..) => "literal",
10281022
NtMeta(..) => "attribute",
@@ -1044,9 +1038,7 @@ impl PartialEq for Nonterminal {
10441038
impl fmt::Debug for Nonterminal {
10451039
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10461040
match *self {
1047-
NtItem(..) => f.pad("NtItem(..)"),
10481041
NtBlock(..) => f.pad("NtBlock(..)"),
1049-
NtStmt(..) => f.pad("NtStmt(..)"),
10501042
NtExpr(..) => f.pad("NtExpr(..)"),
10511043
NtLiteral(..) => f.pad("NtLiteral(..)"),
10521044
NtMeta(..) => f.pad("NtMeta(..)"),

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
//! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
1414
//! ownership of the original.
1515
16-
use crate::ast::{AttrStyle, StmtKind};
16+
use crate::ast::AttrStyle;
1717
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
1818
use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
1919
use crate::AttrVec;
@@ -449,6 +449,16 @@ impl TokenStream {
449449
TokenStream::new(vec![TokenTree::token_alone(kind, span)])
450450
}
451451

452+
/// Create a token stream containing a single `Delimited`.
453+
pub fn delimited(
454+
span: DelimSpan,
455+
spacing: DelimSpacing,
456+
delim: Delimiter,
457+
tts: TokenStream,
458+
) -> TokenStream {
459+
TokenStream::new(vec![TokenTree::Delimited(span, spacing, delim, tts)])
460+
}
461+
452462
pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
453463
let Some(tokens) = node.tokens() else {
454464
panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
@@ -466,13 +476,7 @@ impl TokenStream {
466476

467477
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
468478
match nt {
469-
Nonterminal::NtItem(item) => TokenStream::from_ast(item),
470479
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
471-
Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
472-
// FIXME: Properly collect tokens for empty statements.
473-
TokenStream::token_alone(token::Semi, stmt.span)
474-
}
475-
Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
476480
Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
477481
Nonterminal::NtPath(path) => TokenStream::from_ast(path),
478482
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -856,9 +856,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
856856
token::NtExpr(e) => self.expr_to_string(e),
857857
token::NtMeta(e) => self.attr_item_to_string(e),
858858
token::NtPath(e) => self.path_to_string(e),
859-
token::NtItem(e) => self.item_to_string(e),
860859
token::NtBlock(e) => self.block_to_string(e),
861-
token::NtStmt(e) => self.stmt_to_string(e),
862860
token::NtLiteral(e) => self.expr_to_string(e),
863861
}
864862
}

‎compiler/rustc_expand/src/base.rs

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use crate::module::DirOwnership;
55

66
use rustc_ast::attr::MarkedAttrs;
77
use rustc_ast::ptr::P;
8-
use rustc_ast::token::Nonterminal;
8+
use rustc_ast::token::NonterminalKind;
99
use rustc_ast::tokenstream::TokenStream;
1010
use rustc_ast::visit::{AssocCtxt, Visitor};
1111
use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind};
@@ -1336,13 +1336,13 @@ pub fn parse_macro_name_and_helper_attrs(
13361336
/// asserts in old versions of those crates and their wide use in the ecosystem.
13371337
/// See issue #73345 for more details.
13381338
/// FIXME(#73933): Remove this eventually.
1339-
fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) -> bool {
1339+
fn pretty_printing_compatibility_hack(item: &Item, psess: &ParseSess) -> bool {
13401340
let name = item.ident.name;
13411341
if name == sym::ProceduralMasqueradeDummyType {
13421342
if let ast::ItemKind::Enum(enum_def, _) = &item.kind {
13431343
if let [variant] = &*enum_def.variants {
13441344
if variant.ident.name == sym::Input {
1345-
let filename = sess.source_map().span_to_filename(item.ident.span);
1345+
let filename = psess.source_map().span_to_filename(item.ident.span);
13461346
if let FileName::Real(real) = filename {
13471347
if let Some(c) = real
13481348
.local_path()
@@ -1366,7 +1366,7 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) -> bool {
13661366
if crate_matches {
13671367
// FIXME: make this translatable
13681368
#[allow(rustc::untranslatable_diagnostic)]
1369-
sess.psess.buffer_lint_with_diagnostic(
1369+
psess.buffer_lint_with_diagnostic(
13701370
PROC_MACRO_BACK_COMPAT,
13711371
item.ident.span,
13721372
ast::CRATE_NODE_ID,
@@ -1387,7 +1387,7 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) -> bool {
13871387
false
13881388
}
13891389

1390-
pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &Session) -> bool {
1390+
pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, psess: &ParseSess) -> bool {
13911391
let item = match ann {
13921392
Annotatable::Item(item) => item,
13931393
Annotatable::Stmt(stmt) => match &stmt.kind {
@@ -1396,17 +1396,34 @@ pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &S
13961396
},
13971397
_ => return false,
13981398
};
1399-
pretty_printing_compatibility_hack(item, sess)
1399+
pretty_printing_compatibility_hack(item, psess)
14001400
}
14011401

1402-
pub(crate) fn nt_pretty_printing_compatibility_hack(nt: &Nonterminal, sess: &Session) -> bool {
1403-
let item = match nt {
1404-
Nonterminal::NtItem(item) => item,
1405-
Nonterminal::NtStmt(stmt) => match &stmt.kind {
1406-
ast::StmtKind::Item(item) => item,
1407-
_ => return false,
1408-
},
1402+
pub(crate) fn stream_pretty_printing_compatibility_hack(
1403+
kind: NonterminalKind,
1404+
stream: &TokenStream,
1405+
psess: &ParseSess,
1406+
) -> bool {
1407+
let item = match kind {
1408+
NonterminalKind::Item => {
1409+
let mut parser = parser::Parser::new(psess, stream.clone(), None);
1410+
let Ok(parser::ParseNtResult::Item(item)) = parser.parse_nonterminal(kind) else {
1411+
panic!("failed to reparse");
1412+
};
1413+
item
1414+
}
1415+
NonterminalKind::Stmt => {
1416+
// njn: reparsing and then checking for StmtKind::Item sucks, hmm
1417+
let mut parser = parser::Parser::new(psess, stream.clone(), None);
1418+
let Ok(parser::ParseNtResult::Stmt(stmt)) = parser.parse_nonterminal(kind) else {
1419+
panic!("failed to reparse");
1420+
};
1421+
match &stmt.kind {
1422+
ast::StmtKind::Item(item) => item.clone(),
1423+
_ => return false,
1424+
}
1425+
}
14091426
_ => return false,
14101427
};
1411-
pretty_printing_compatibility_hack(item, sess)
1428+
pretty_printing_compatibility_hack(&item, psess)
14121429
}

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use crate::mbe::{self, KleeneOp, MetaVarExpr};
88
use rustc_ast::mut_visit::{self, MutVisitor};
99
use rustc_ast::token::{self, Delimiter, InvisibleOrigin, NonterminalKind, Token, TokenKind};
1010
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
11+
use rustc_ast::StmtKind;
1112
use rustc_data_structures::fx::FxHashMap;
1213
use rustc_errors::{pluralize, Diag, PResult};
1314
use rustc_parse::parser::ParseNtResult;
@@ -283,6 +284,18 @@ pub(super) fn transcribe<'a>(
283284
let kind = token::NtLifetime(*ident);
284285
TokenTree::token_alone(kind, sp)
285286
}
287+
MatchedSingle(ParseNtResult::Item(ref item)) => {
288+
mk_delimited(NonterminalKind::Item, TokenStream::from_ast(item))
289+
}
290+
MatchedSingle(ParseNtResult::Stmt(ref stmt)) => {
291+
let stream = if let StmtKind::Empty = stmt.kind {
292+
// FIXME: Properly collect tokens for empty statements.
293+
TokenStream::token_alone(token::Semi, stmt.span)
294+
} else {
295+
TokenStream::from_ast(stmt)
296+
};
297+
mk_delimited(NonterminalKind::Stmt, stream)
298+
}
286299
MatchedSingle(ParseNtResult::PatParam(ref pat, inferred)) => mk_delimited(
287300
NonterminalKind::PatParam { inferred: *inferred },
288301
TokenStream::from_ast(pat),

‎compiler/rustc_expand/src/proc_macro.rs

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@ use crate::proc_macro_server;
44

55
use rustc_ast as ast;
66
use rustc_ast::ptr::P;
7-
use rustc_ast::token;
8-
use rustc_ast::tokenstream::TokenStream;
9-
use rustc_data_structures::sync::Lrc;
7+
use rustc_ast::token::{Delimiter, InvisibleOrigin, NonterminalKind};
8+
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream};
109
use rustc_errors::ErrorGuaranteed;
1110
use rustc_parse::parser::ForceCollect;
1211
use rustc_session::config::ProcMacroExecutionStrategy;
@@ -120,14 +119,25 @@ impl MultiItemModifier for DeriveProcMacro {
120119
// We need special handling for statement items
121120
// (e.g. `fn foo() { #[derive(Debug)] struct Bar; }`)
122121
let is_stmt = matches!(item, Annotatable::Stmt(..));
123-
let hack = crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess);
122+
let hack = crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess.psess);
124123
let input = if hack {
125-
let nt = match item {
126-
Annotatable::Item(item) => token::NtItem(item),
127-
Annotatable::Stmt(stmt) => token::NtStmt(stmt),
124+
let delim_span = DelimSpan::from_single(DUMMY_SP);
125+
let delim_spacing = DelimSpacing::new(Spacing::Alone, Spacing::Alone);
126+
match item {
127+
Annotatable::Item(item) => TokenStream::delimited(
128+
delim_span,
129+
delim_spacing,
130+
Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Item)),
131+
TokenStream::from_ast(&item),
132+
),
133+
Annotatable::Stmt(stmt) => TokenStream::delimited(
134+
delim_span,
135+
delim_spacing,
136+
Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Stmt)),
137+
TokenStream::from_ast(&stmt),
138+
),
128139
_ => unreachable!(),
129-
};
130-
TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP)
140+
}
131141
} else {
132142
item.to_tokens()
133143
};

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -113,17 +113,32 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
113113

114114
while let Some(tree) = cursor.next() {
115115
let (Token { kind, span }, joint) = match tree.clone() {
116-
tokenstream::TokenTree::Delimited(span, _, delim, tts) => {
117-
let delimiter = pm::Delimiter::from_internal(delim);
118-
trees.push(TokenTree::Group(Group {
119-
delimiter,
120-
stream: Some(tts),
121-
span: DelimSpan {
122-
open: span.open,
123-
close: span.close,
124-
entire: span.entire(),
125-
},
126-
}));
116+
tokenstream::TokenTree::Delimited(span, _, delim, stream) => {
117+
// A hack used to pass AST fragments to attribute and derive
118+
// macros as a single nonterminal token instead of a token
119+
// stream. Such token needs to be "unwrapped" and not
120+
// represented as a delimited group.
121+
// FIXME: It needs to be removed, but there are some
122+
// compatibility issues (see #73345).
123+
if let Delimiter::Invisible(InvisibleOrigin::MetaVar(kind)) = delim
124+
&& crate::base::stream_pretty_printing_compatibility_hack(
125+
kind,
126+
&stream,
127+
rustc.psess(),
128+
)
129+
{
130+
trees.extend(Self::from_internal((stream, rustc)));
131+
} else {
132+
trees.push(TokenTree::Group(Group {
133+
delimiter: pm::Delimiter::from_internal(delim),
134+
stream: Some(stream),
135+
span: DelimSpan {
136+
open: span.open,
137+
close: span.close,
138+
entire: span.entire(),
139+
},
140+
}));
141+
}
127142
continue;
128143
}
129144
tokenstream::TokenTree::Token(token, spacing) => {
@@ -276,21 +291,11 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
276291

277292
Interpolated(nt) => {
278293
let stream = TokenStream::from_nonterminal_ast(&nt);
279-
// A hack used to pass AST fragments to attribute and derive
280-
// macros as a single nonterminal token instead of a token
281-
// stream. Such token needs to be "unwrapped" and not
282-
// represented as a delimited group.
283-
// FIXME: It needs to be removed, but there are some
284-
// compatibility issues (see #73345).
285-
if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.ecx.sess) {
286-
trees.extend(Self::from_internal((stream, rustc)));
287-
} else {
288-
trees.push(TokenTree::Group(Group {
289-
delimiter: pm::Delimiter::None,
290-
stream: Some(stream),
291-
span: DelimSpan::from_single(span),
292-
}))
293-
}
294+
trees.push(TokenTree::Group(Group {
295+
delimiter: pm::Delimiter::None,
296+
stream: Some(stream),
297+
span: DelimSpan::from_single(span),
298+
}))
294299
}
295300

296301
OpenDelim(..) | CloseDelim(..) => unreachable!(),

‎compiler/rustc_parse/src/parser/item.rs

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
use super::diagnostics::{dummy_arg, ConsumeClosingDelim};
22
use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
33
use super::{
4-
AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, Trailing, TrailingToken,
4+
AttrWrapper, FollowedByType, ForceCollect, ParseNtResult, Parser, PathStyle, Recovered,
5+
Trailing, TrailingToken,
56
};
67
use crate::errors::{self, MacroExpandsToAdtField};
78
use crate::fluent_generated as fluent;
8-
use crate::maybe_whole;
9+
use crate::maybe_reparse_metavar_seq;
910
use ast::token::IdentIsRaw;
1011
use rustc_ast::ast::*;
1112
use rustc_ast::ptr::P;
@@ -115,10 +116,16 @@ impl<'a> Parser<'a> {
115116
fn_parse_mode: FnParseMode,
116117
force_collect: ForceCollect,
117118
) -> PResult<'a, Option<Item>> {
118-
maybe_whole!(self, NtItem, |item| {
119+
if let Some(mut item) = maybe_reparse_metavar_seq!(
120+
self,
121+
NonterminalKind::Item,
122+
NonterminalKind::Item,
123+
ParseNtResult::Item(item),
124+
item
125+
) {
119126
attrs.prepend_to_nt_inner(&mut item.attrs);
120-
Some(item.into_inner())
121-
});
127+
return Ok(Some(item.into_inner()));
128+
}
122129

123130
let item =
124131
self.collect_tokens_trailing_token(attrs, force_collect, |this: &mut Self, attrs| {

0 commit comments

Comments
 (0)