Remove NtItem and NtStmt.

This involves replacing `nt_pretty_printing_compatibility_hack` with
`stream_pretty_printing_compatibility_hack`.

The handling of statements in `transcribe` is slightly different to
other nonterminal kinds, due to the lack of `from_ast` implementation
for empty statements.

Notable test changes:
- `tests/ui/macros/nonterminal-matching.rs` now passes. Removal of
  `Token::Interpolated` will remove the "captured metavariables except
  for `:tt`, `:ident` and `:lifetime` cannot be compared to other
  tokens" restriction.

- `tests/ui/proc-macro/expand-to-derive.rs`: the diff looks large but
  the only difference is the insertion of a single invisible-delimited
  group. I hope this doesn't cause any new `ProceduralMasquerade`-type
  problems.

Author: nnethercote

compiler/rustc_ast/src/ast_traits.rs

@@ -231,8 +231,6 @@ impl HasTokens for Attribute {
 impl HasTokens for Nonterminal {
     fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match self {
-            Nonterminal::NtItem(item) => item.tokens(),
-            Nonterminal::NtStmt(stmt) => stmt.tokens(),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
             Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
             Nonterminal::NtPath(path) => path.tokens(),
@@ -242,8 +240,6 @@ impl HasTokens for Nonterminal {
     }
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         match self {
-            Nonterminal::NtItem(item) => item.tokens_mut(),
-            Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
             Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
             Nonterminal::NtPath(path) => path.tokens_mut(),

compiler/rustc_ast/src/mut_visit.rs

@@ -823,19 +823,7 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
 // multiple items there....
 pub fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
     match nt {
-        token::NtItem(item) => visit_clobber(item, |item| {
-            // This is probably okay, because the only visitors likely to
-            // peek inside interpolated nodes will be renamings/markings,
-            // which map single items to single items.
-            vis.flat_map_item(item).expect_one("expected visitor to produce exactly one item")
-        }),
         token::NtBlock(block) => vis.visit_block(block),
-        token::NtStmt(stmt) => visit_clobber(stmt, |stmt| {
-            // See reasoning above.
-            stmt.map(|stmt| {
-                vis.flat_map_stmt(stmt).expect_one("expected visitor to produce exactly one item")
-            })
-        }),
         token::NtExpr(expr) => vis.visit_expr(expr),
         token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
         token::NtLifetime(ident) => vis.visit_ident(ident),

compiler/rustc_ast/src/token.rs

@@ -924,9 +924,7 @@ impl PartialEq<TokenKind> for Token {
 #[derive(Clone, Encodable, Decodable)]
 /// For interpolation during macro expansion.
 pub enum Nonterminal {
-    NtItem(P<ast::Item>),
     NtBlock(P<ast::Block>),
-    NtStmt(P<ast::Stmt>),
     NtExpr(P<ast::Expr>),
     NtIdent(Ident, IdentIsRaw),
     NtLifetime(Ident),
@@ -1017,9 +1015,7 @@ impl fmt::Display for NonterminalKind {
 impl Nonterminal {
     pub fn use_span(&self) -> Span {
         match self {
-            NtItem(item) => item.span,
             NtBlock(block) => block.span,
-            NtStmt(stmt) => stmt.span,
             NtExpr(expr) | NtLiteral(expr) => expr.span,
             NtIdent(ident, _) | NtLifetime(ident) => ident.span,
             NtMeta(attr_item) => attr_item.span(),
@@ -1029,9 +1025,7 @@ impl Nonterminal {
 
     pub fn descr(&self) -> &'static str {
         match self {
-            NtItem(..) => "item",
             NtBlock(..) => "block",
-            NtStmt(..) => "statement",
             NtExpr(..) => "expression",
             NtLiteral(..) => "literal",
             NtIdent(..) => "identifier",
@@ -1061,9 +1055,7 @@ impl PartialEq for Nonterminal {
 impl fmt::Debug for Nonterminal {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
-            NtItem(..) => f.pad("NtItem(..)"),
             NtBlock(..) => f.pad("NtBlock(..)"),
-            NtStmt(..) => f.pad("NtStmt(..)"),
             NtExpr(..) => f.pad("NtExpr(..)"),
             NtIdent(..) => f.pad("NtIdent(..)"),
             NtLiteral(..) => f.pad("NtLiteral(..)"),

compiler/rustc_ast/src/tokenstream.rs

@@ -13,7 +13,7 @@
 //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
 //! ownership of the original.
 
-use crate::ast::{AttrStyle, StmtKind};
+use crate::ast::AttrStyle;
 use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
 use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
 use crate::AttrVec;
@@ -450,6 +450,16 @@ impl TokenStream {
         TokenStream::new(vec![TokenTree::token_alone(kind, span)])
     }
 
+    /// Create a token stream containing a single `Delimited`.
+    pub fn delimited(
+        span: DelimSpan,
+        spacing: DelimSpacing,
+        delim: Delimiter,
+        tts: TokenStream,
+    ) -> TokenStream {
+        TokenStream::new(vec![TokenTree::Delimited(span, spacing, delim, tts)])
+    }
+
     pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
         let Some(tokens) = node.tokens() else {
             panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
@@ -473,13 +483,7 @@ impl TokenStream {
             Nonterminal::NtLifetime(ident) => {
                 TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
             }
-            Nonterminal::NtItem(item) => TokenStream::from_ast(item),
             Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
-            Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
-                // FIXME: Properly collect tokens for empty statements.
-                TokenStream::token_alone(token::Semi, stmt.span)
-            }
-            Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
             Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
             Nonterminal::NtPath(path) => TokenStream::from_ast(path),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -847,9 +847,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::NtExpr(e) => self.expr_to_string(e),
             token::NtMeta(e) => self.attr_item_to_string(e),
             token::NtPath(e) => self.path_to_string(e),
-            token::NtItem(e) => self.item_to_string(e),
             token::NtBlock(e) => self.block_to_string(e),
-            token::NtStmt(e) => self.stmt_to_string(e),
             &token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(),
             token::NtLifetime(e) => e.to_string(),
             token::NtLiteral(e) => self.expr_to_string(e),

compiler/rustc_expand/src/base.rs

@@ -5,7 +5,7 @@ use crate::module::DirOwnership;
 
 use rustc_ast::attr::MarkedAttrs;
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Nonterminal};
+use rustc_ast::token::{self, NonterminalKind};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{AssocCtxt, Visitor};
 use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind};
@@ -1524,13 +1524,13 @@ pub fn parse_macro_name_and_helper_attrs(
 /// asserts in old versions of those crates and their wide use in the ecosystem.
 /// See issue #73345 for more details.
 /// FIXME(#73933): Remove this eventually.
-fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) -> bool {
+fn pretty_printing_compatibility_hack(item: &Item, psess: &ParseSess) -> bool {
     let name = item.ident.name;
     if name == sym::ProceduralMasqueradeDummyType {
         if let ast::ItemKind::Enum(enum_def, _) = &item.kind {
             if let [variant] = &*enum_def.variants {
                 if variant.ident.name == sym::Input {
-                    let filename = sess.source_map().span_to_filename(item.ident.span);
+                    let filename = psess.source_map().span_to_filename(item.ident.span);
                     if let FileName::Real(real) = filename {
                         if let Some(c) = real
                             .local_path()
@@ -1554,7 +1554,7 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) -> bool {
                             if crate_matches {
                                 // FIXME: make this translatable
                                 #[allow(rustc::untranslatable_diagnostic)]
-                                sess.psess.buffer_lint_with_diagnostic(
+                                psess.buffer_lint_with_diagnostic(
                                         PROC_MACRO_BACK_COMPAT,
                                         item.ident.span,
                                         ast::CRATE_NODE_ID,
@@ -1575,7 +1575,7 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) -> bool {
     false
 }
 
-pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &Session) -> bool {
+pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, psess: &ParseSess) -> bool {
     let item = match ann {
         Annotatable::Item(item) => item,
         Annotatable::Stmt(stmt) => match &stmt.kind {
@@ -1584,17 +1584,34 @@ pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &S
         },
         _ => return false,
     };
-    pretty_printing_compatibility_hack(item, sess)
+    pretty_printing_compatibility_hack(item, psess)
 }
 
-pub(crate) fn nt_pretty_printing_compatibility_hack(nt: &Nonterminal, sess: &Session) -> bool {
-    let item = match nt {
-        Nonterminal::NtItem(item) => item,
-        Nonterminal::NtStmt(stmt) => match &stmt.kind {
-            ast::StmtKind::Item(item) => item,
-            _ => return false,
-        },
+pub(crate) fn stream_pretty_printing_compatibility_hack(
+    kind: NonterminalKind,
+    stream: &TokenStream,
+    psess: &ParseSess,
+) -> bool {
+    let item = match kind {
+        NonterminalKind::Item => {
+            let mut parser = parser::Parser::new(psess, stream.clone(), None);
+            let Ok(parser::ParseNtResult::Item(item)) = parser.parse_nonterminal(kind) else {
+                panic!("failed to reparse");
+            };
+            item
+        }
+        NonterminalKind::Stmt => {
+            // njn: reparsing and then checking for StmtKind::Item sucks, hmm
+            let mut parser = parser::Parser::new(psess, stream.clone(), None);
+            let Ok(parser::ParseNtResult::Stmt(stmt)) = parser.parse_nonterminal(kind) else {
+                panic!("failed to reparse");
+            };
+            match &stmt.kind {
+                ast::StmtKind::Item(item) => item.clone(),
+                _ => return false,
+            }
+        }
         _ => return false,
     };
-    pretty_printing_compatibility_hack(item, sess)
+    pretty_printing_compatibility_hack(&item, psess)
 }

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -8,6 +8,7 @@ use crate::mbe::{self, KleeneOp, MetaVarExpr};
 use rustc_ast::mut_visit::{self, MutVisitor};
 use rustc_ast::token::{self, Delimiter, InvisibleOrigin, NonterminalKind, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
+use rustc_ast::StmtKind;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{pluralize, Diag, PResult};
 use rustc_parse::parser::ParseNtResult;
@@ -266,6 +267,18 @@ pub(super) fn transcribe<'a>(
                             // without wrapping them into groups.
                             maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker)
                         }
+                        MatchedSingle(ParseNtResult::Item(ref item)) => {
+                            mk_delimited(NonterminalKind::Item, TokenStream::from_ast(item))
+                        }
+                        MatchedSingle(ParseNtResult::Stmt(ref stmt)) => {
+                            let stream = if let StmtKind::Empty = stmt.kind {
+                                // FIXME: Properly collect tokens for empty statements.
+                                TokenStream::token_alone(token::Semi, stmt.span)
+                            } else {
+                                TokenStream::from_ast(stmt)
+                            };
+                            mk_delimited(NonterminalKind::Stmt, stream)
+                        }
                         MatchedSingle(ParseNtResult::PatParam(ref pat, inferred)) => mk_delimited(
                             NonterminalKind::PatParam { inferred: *inferred },
                             TokenStream::from_ast(pat),

compiler/rustc_expand/src/proc_macro.rs

@@ -4,9 +4,8 @@ use crate::proc_macro_server;
 
 use rustc_ast as ast;
 use rustc_ast::ptr::P;
-use rustc_ast::token;
-use rustc_ast::tokenstream::TokenStream;
-use rustc_data_structures::sync::Lrc;
+use rustc_ast::token::{Delimiter, InvisibleOrigin, NonterminalKind};
+use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream};
 use rustc_errors::ErrorGuaranteed;
 use rustc_parse::parser::ForceCollect;
 use rustc_session::config::ProcMacroExecutionStrategy;
@@ -120,14 +119,25 @@ impl MultiItemModifier for DeriveProcMacro {
         // We need special handling for statement items
         // (e.g. `fn foo() { #[derive(Debug)] struct Bar; }`)
         let is_stmt = matches!(item, Annotatable::Stmt(..));
-        let hack = crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess);
+        let hack = crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess.psess);
         let input = if hack {
-            let nt = match item {
-                Annotatable::Item(item) => token::NtItem(item),
-                Annotatable::Stmt(stmt) => token::NtStmt(stmt),
+            let delim_span = DelimSpan::from_single(DUMMY_SP);
+            let delim_spacing = DelimSpacing::new(Spacing::Alone, Spacing::Alone); // njn: ?
+            match item {
+                Annotatable::Item(item) => TokenStream::delimited(
+                    delim_span,
+                    delim_spacing,
+                    Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Item)),
+                    TokenStream::from_ast(&item),
+                ),
+                Annotatable::Stmt(stmt) => TokenStream::delimited(
+                    delim_span,
+                    delim_spacing,
+                    Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Stmt)),
+                    TokenStream::from_ast(&stmt),
+                ),
                 _ => unreachable!(),
-            };
-            TokenStream::token_alone(token::Interpolated(Lrc::new((nt, span))), DUMMY_SP)
+            }
         } else {
             item.to_tokens()
         };

compiler/rustc_expand/src/proc_macro_server.rs

@@ -113,17 +113,32 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
 
         while let Some(tree) = cursor.next() {
             let (Token { kind, span }, joint) = match tree.clone() {
-                tokenstream::TokenTree::Delimited(span, _, delim, tts) => {
-                    let delimiter = pm::Delimiter::from_internal(delim);
-                    trees.push(TokenTree::Group(Group {
-                        delimiter,
-                        stream: Some(tts),
-                        span: DelimSpan {
-                            open: span.open,
-                            close: span.close,
-                            entire: span.entire(),
-                        },
-                    }));
+                tokenstream::TokenTree::Delimited(span, _, delim, stream) => {
+                    // A hack used to pass AST fragments to attribute and derive
+                    // macros as a single nonterminal token instead of a token
+                    // stream. Such token needs to be "unwrapped" and not
+                    // represented as a delimited group.
+                    // FIXME: It needs to be removed, but there are some
+                    // compatibility issues (see #73345).
+                    if let Delimiter::Invisible(InvisibleOrigin::MetaVar(kind)) = delim
+                        && crate::base::stream_pretty_printing_compatibility_hack(
+                            kind,
+                            &stream,
+                            rustc.psess(),
+                        )
+                    {
+                        trees.extend(Self::from_internal((stream, rustc)));
+                    } else {
+                        trees.push(TokenTree::Group(Group {
+                            delimiter: pm::Delimiter::from_internal(delim),
+                            stream: Some(stream),
+                            span: DelimSpan {
+                                open: span.open,
+                                close: span.close,
+                                entire: span.entire(),
+                            },
+                        }));
+                    }
                     continue;
                 }
                 tokenstream::TokenTree::Token(token, spacing) => {
@@ -269,21 +284,11 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
 
                 Interpolated(nt) => {
                     let stream = TokenStream::from_nonterminal_ast(&nt.0);
-                    // A hack used to pass AST fragments to attribute and derive
-                    // macros as a single nonterminal token instead of a token
-                    // stream. Such token needs to be "unwrapped" and not
-                    // represented as a delimited group.
-                    // FIXME: It needs to be removed, but there are some
-                    // compatibility issues (see #73345).
-                    if crate::base::nt_pretty_printing_compatibility_hack(&nt.0, rustc.ecx.sess) {
-                        trees.extend(Self::from_internal((stream, rustc)));
-                    } else {
-                        trees.push(TokenTree::Group(Group {
-                            delimiter: pm::Delimiter::None,
-                            stream: Some(stream),
-                            span: DelimSpan::from_single(span),
-                        }))
-                    }
+                    trees.push(TokenTree::Group(Group {
+                        delimiter: pm::Delimiter::None,
+                        stream: Some(stream),
+                        span: DelimSpan::from_single(span),
+                    }))
                 }
 
                 // njn: covers invis-delim

compiler/rustc_parse/src/parser/item.rs

@@ -1,12 +1,12 @@
 use super::diagnostics::{dummy_arg, ConsumeClosingDelim};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{
-    AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, Recovered, Trailing,
-    TrailingToken,
+    AttrWrapper, FollowedByType, ForceCollect, ParseNtResult, Parser, PathStyle, Recovered,
+    Trailing, TrailingToken,
 };
 use crate::errors::{self, MacroExpandsToAdtField};
 use crate::fluent_generated as fluent;
-use crate::maybe_whole;
+use crate::maybe_reparse_metavar_seq;
 use ast::token::IdentIsRaw;
 use rustc_ast::ast::*;
 use rustc_ast::ptr::P;
@@ -116,10 +116,16 @@ impl<'a> Parser<'a> {
         fn_parse_mode: FnParseMode,
         force_collect: ForceCollect,
     ) -> PResult<'a, Option<Item>> {
-        maybe_whole!(self, NtItem, |item| {
+        if let Some(mut item) = maybe_reparse_metavar_seq!(
+            self,
+            NonterminalKind::Item,
+            NonterminalKind::Item,
+            ParseNtResult::Item(item),
+            item
+        ) {
             attrs.prepend_to_nt_inner(&mut item.attrs);
-            Some(item.into_inner())
-        });
+            return Ok(Some(item.into_inner()));
+        }
 
         let item =
             self.collect_tokens_trailing_token(attrs, force_collect, |this: &mut Self, attrs| {