Rollup merge of rust-lang#69838 - Centril:expand-module, r=petrochenkov

Expansion-driven outline module parsing

After this PR, the parser will not do any conditional compilation or loading of external module files when `mod foo;` is encountered. Instead, the parser only leaves `mod foo;` in place in the AST, with no items filled in. Expansion later kicks in and will load the actual files and do the parsing. This entails that the following is now valid:

```rust
#[cfg(FALSE)]
mod foo {
    mod bar {
        mod baz; // `foo/bar/baz.rs` doesn't exist, but no error!
    }
}
```

Fixes rust-lang#64197.

r? @petrochenkov

Author: Centril

src/libpanic_unwind/emcc.rs

@@ -6,8 +6,6 @@
 //! Emscripten's runtime always implements those APIs and does not
 //! implement libunwind.
 
-#![allow(private_no_mangle_fns)]
-
 use alloc::boxed::Box;
 use core::any::Any;
 use core::mem;

src/libpanic_unwind/gcc.rs

@@ -36,8 +36,6 @@
 //! Once stack has been unwound down to the handler frame level, unwinding stops
 //! and the last personality routine transfers control to the catch block.
 
-#![allow(private_no_mangle_fns)]
-
 use alloc::boxed::Box;
 use core::any::Any;
 

src/libpanic_unwind/seh.rs

@@ -45,7 +45,6 @@
 //! [llvm]: http://llvm.org/docs/ExceptionHandling.html#background-on-windows-exceptions
 
 #![allow(nonstandard_style)]
-#![allow(private_no_mangle_fns)]
 
 use alloc::boxed::Box;
 use core::any::Any;

src/librustc_ast/ast.rs

@@ -2153,7 +2153,7 @@ impl FnRetTy {
 /// Module declaration.
 ///
 /// E.g., `mod foo;` or `mod foo { .. }`.
-#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
+#[derive(Clone, RustcEncodable, RustcDecodable, Debug, Default)]
 pub struct Mod {
     /// A span from the first token past `{` to the last token until `}`.
     /// For `mod foo;`, the inner span ranges from the first token

src/librustc_builtin_macros/proc_macro_harness.rs

@@ -59,7 +59,7 @@ pub fn inject(
     handler: &rustc_errors::Handler,
 ) -> ast::Crate {
     let ecfg = ExpansionConfig::default("proc_macro".to_string());
-    let mut cx = ExtCtxt::new(sess, ecfg, resolver);
+    let mut cx = ExtCtxt::new(sess, ecfg, resolver, None);
 
     let mut collect = CollectProcMacros {
         macros: Vec::new(),

src/librustc_builtin_macros/source_util.rs

@@ -5,7 +5,7 @@ use rustc_ast::tokenstream::TokenStream;
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::{self, *};
 use rustc_expand::panictry;
-use rustc_parse::{self, new_sub_parser_from_file, parser::Parser, DirectoryOwnership};
+use rustc_parse::{self, new_sub_parser_from_file, parser::Parser};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
 use rustc_span::symbol::Symbol;
 use rustc_span::{self, Pos, Span};
@@ -108,8 +108,7 @@ pub fn expand_include<'cx>(
             return DummyResult::any(sp);
         }
     };
-    let directory_ownership = DirectoryOwnership::Owned { relative: None };
-    let p = new_sub_parser_from_file(cx.parse_sess(), &file, directory_ownership, None, sp);
+    let p = new_sub_parser_from_file(cx.parse_sess(), &file, None, sp);
 
     struct ExpandResult<'a> {
         p: Parser<'a>,

src/librustc_builtin_macros/standard_library_imports.rs

@@ -39,7 +39,7 @@ pub fn inject(
     let call_site = DUMMY_SP.with_call_site_ctxt(expn_id);
 
     let ecfg = ExpansionConfig::default("std_lib_injection".to_string());
-    let cx = ExtCtxt::new(sess, ecfg, resolver);
+    let cx = ExtCtxt::new(sess, ecfg, resolver, None);
 
     // .rev() to preserve ordering above in combination with insert(0, ...)
     for &name in names.iter().rev() {

src/librustc_builtin_macros/test_harness.rs

@@ -202,7 +202,7 @@ fn generate_test_harness(
     let mut econfig = ExpansionConfig::default("test".to_string());
     econfig.features = Some(features);
 
-    let ext_cx = ExtCtxt::new(sess, econfig, resolver);
+    let ext_cx = ExtCtxt::new(sess, econfig, resolver, None);
 
     let expn_id = ext_cx.resolver.expansion_for_ast_pass(
         DUMMY_SP,

src/librustc_expand/base.rs

@@ -1,4 +1,5 @@
 use crate::expand::{self, AstFragment, Invocation};
+use crate::module::DirectoryOwnership;
 
 use rustc_ast::ast::{self, Attribute, Name, NodeId, PatKind};
 use rustc_ast::mut_visit::{self, MutVisitor};
@@ -10,7 +11,7 @@ use rustc_attr::{self as attr, Deprecation, HasAttrs, Stability};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{DiagnosticBuilder, DiagnosticId};
-use rustc_parse::{self, parser, DirectoryOwnership, MACRO_ARGUMENTS};
+use rustc_parse::{self, parser, MACRO_ARGUMENTS};
 use rustc_session::parse::ParseSess;
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::{AstPass, ExpnData, ExpnId, ExpnKind};
@@ -925,19 +926,23 @@ pub struct ExtCtxt<'a> {
     pub resolver: &'a mut dyn Resolver,
     pub current_expansion: ExpansionData,
     pub expansions: FxHashMap<Span, Vec<String>>,
+    /// Called directly after having parsed an external `mod foo;` in expansion.
+    pub(super) extern_mod_loaded: Option<&'a dyn Fn(&ast::Crate)>,
 }
 
 impl<'a> ExtCtxt<'a> {
     pub fn new(
         parse_sess: &'a ParseSess,
         ecfg: expand::ExpansionConfig<'a>,
         resolver: &'a mut dyn Resolver,
+        extern_mod_loaded: Option<&'a dyn Fn(&ast::Crate)>,
     ) -> ExtCtxt<'a> {
         ExtCtxt {
             parse_sess,
             ecfg,
-            root_path: PathBuf::new(),
             resolver,
+            extern_mod_loaded,
+            root_path: PathBuf::new(),
             current_expansion: ExpansionData {
                 id: ExpnId::root(),
                 depth: 0,

src/librustc_parse/config.rs renamed to src/librustc_expand/config.rs

@@ -1,14 +1,5 @@
-//! Process the potential `cfg` attributes on a module.
-//! Also determine if the module should be included in this configuration.
-//!
-//! This module properly belongs in rustc_expand, but for now it's tied into
-//! parsing, so we leave it here to avoid complicated out-of-line dependencies.
-//!
-//! A principled solution to this wrong location would be to implement [#64197].
-//!
-//! [#64197]: https://github.com/rust-lang/rust/issues/64197
-
-use crate::{parse_in, validate_attr};
+//! Conditional compilation stripping.
+
 use rustc_ast::ast::{self, AttrItem, Attribute, MetaItem};
 use rustc_ast::attr::HasAttrs;
 use rustc_ast::mut_visit::*;
@@ -21,6 +12,7 @@ use rustc_feature::{Feature, Features, State as FeatureState};
 use rustc_feature::{
     ACCEPTED_FEATURES, ACTIVE_FEATURES, REMOVED_FEATURES, STABLE_REMOVED_FEATURES,
 };
+use rustc_parse::{parse_in, validate_attr};
 use rustc_session::parse::{feature_err, ParseSess};
 use rustc_span::edition::{Edition, ALL_EDITIONS};
 use rustc_span::symbol::{sym, Symbol};
@@ -538,12 +530,3 @@ impl<'a> MutVisitor for StripUnconfigured<'a> {
 fn is_cfg(attr: &Attribute) -> bool {
     attr.check_name(sym::cfg)
 }
-
-/// Process the potential `cfg` attributes on a module.
-/// Also determine if the module should be included in this configuration.
-pub fn process_configure_mod(sess: &ParseSess, cfg_mods: bool, attrs: &mut Vec<Attribute>) -> bool {
-    // Don't perform gated feature checking.
-    let mut strip_unconfigured = StripUnconfigured { sess, features: None };
-    strip_unconfigured.process_cfg_attrs(attrs);
-    !cfg_mods || strip_unconfigured.in_cfg(&attrs)
-}

src/librustc_expand/expand.rs

@@ -1,7 +1,9 @@
 use crate::base::*;
 use crate::config::StripUnconfigured;
+use crate::configure;
 use crate::hygiene::{ExpnData, ExpnId, ExpnKind, SyntaxContext};
 use crate::mbe::macro_rules::annotate_err_with_kind;
+use crate::module::{parse_external_mod, push_directory, Directory, DirectoryOwnership};
 use crate::placeholders::{placeholder, PlaceholderExpander};
 use crate::proc_macro::collect_derives;
 
@@ -17,10 +19,8 @@ use rustc_ast_pretty::pprust;
 use rustc_attr::{self as attr, is_builtin_attr, HasAttrs};
 use rustc_errors::{Applicability, FatalError, PResult};
 use rustc_feature::Features;
-use rustc_parse::configure;
 use rustc_parse::parser::Parser;
 use rustc_parse::validate_attr;
-use rustc_parse::DirectoryOwnership;
 use rustc_session::lint::builtin::UNUSED_DOC_COMMENTS;
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_session::parse::{feature_err, ParseSess};
@@ -1427,59 +1427,83 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
                 .make_items();
         }
 
+        let mut attrs = mem::take(&mut item.attrs); // We do this to please borrowck.
+        let ident = item.ident;
+        let span = item.span;
+
         match item.kind {
             ast::ItemKind::MacCall(..) => {
+                item.attrs = attrs;
                 self.check_attributes(&item.attrs);
                 item.and_then(|item| match item.kind {
                     ItemKind::MacCall(mac) => self
-                        .collect(
-                            AstFragmentKind::Items,
-                            InvocationKind::Bang { mac, span: item.span },
-                        )
+                        .collect(AstFragmentKind::Items, InvocationKind::Bang { mac, span })
                         .make_items(),
                     _ => unreachable!(),
                 })
             }
-            ast::ItemKind::Mod(ast::Mod { inner, inline, .. })
-                if item.ident != Ident::invalid() =>
-            {
-                let orig_directory_ownership = self.cx.current_expansion.directory_ownership;
+            ast::ItemKind::Mod(ref mut old_mod @ ast::Mod { .. }) if ident != Ident::invalid() => {
+                let sess = self.cx.parse_sess;
+                let orig_ownership = self.cx.current_expansion.directory_ownership;
                 let mut module = (*self.cx.current_expansion.module).clone();
-                module.mod_path.push(item.ident);
 
-                if inline {
-                    if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) {
-                        self.cx.current_expansion.directory_ownership =
-                            DirectoryOwnership::Owned { relative: None };
-                        module.directory.push(&*path.as_str());
-                    } else {
-                        module.directory.push(&*item.ident.as_str());
-                    }
+                let pushed = &mut false; // Record `parse_external_mod` pushing so we can pop.
+                let dir = Directory { ownership: orig_ownership, path: module.directory };
+                let Directory { ownership, path } = if old_mod.inline {
+                    // Inline `mod foo { ... }`, but we still need to push directories.
+                    item.attrs = attrs;
+                    push_directory(ident, &item.attrs, dir)
                 } else {
-                    let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner);
-                    let mut path = match path {
-                        FileName::Real(path) => path,
-                        other => PathBuf::from(other.to_string()),
+                    // We have an outline `mod foo;` so we need to parse the file.
+                    let (new_mod, dir) =
+                        parse_external_mod(sess, ident, span, dir, &mut attrs, pushed);
+
+                    let krate = ast::Crate {
+                        span: new_mod.inner,
+                        module: new_mod,
+                        attrs,
+                        proc_macros: vec![],
                     };
-                    let directory_ownership = match path.file_name().unwrap().to_str() {
-                        Some("mod.rs") => DirectoryOwnership::Owned { relative: None },
-                        Some(_) => DirectoryOwnership::Owned { relative: Some(item.ident) },
-                        None => DirectoryOwnership::UnownedViaMod,
+                    if let Some(extern_mod_loaded) = self.cx.extern_mod_loaded {
+                        extern_mod_loaded(&krate);
+                    }
+
+                    *old_mod = krate.module;
+                    item.attrs = krate.attrs;
+                    // File can have inline attributes, e.g., `#![cfg(...)]` & co. => Reconfigure.
+                    item = match self.configure(item) {
+                        Some(node) => node,
+                        None => {
+                            if *pushed {
+                                sess.included_mod_stack.borrow_mut().pop();
+                            }
+                            return Default::default();
+                        }
                     };
-                    path.pop();
-                    module.directory = path;
-                    self.cx.current_expansion.directory_ownership = directory_ownership;
-                }
+                    dir
+                };
 
+                // Set the module info before we flat map.
+                self.cx.current_expansion.directory_ownership = ownership;
+                module.directory = path;
+                module.mod_path.push(ident);
                 let orig_module =
                     mem::replace(&mut self.cx.current_expansion.module, Rc::new(module));
+
                 let result = noop_flat_map_item(item, self);
+
+                // Restore the module info.
                 self.cx.current_expansion.module = orig_module;
-                self.cx.current_expansion.directory_ownership = orig_directory_ownership;
+                self.cx.current_expansion.directory_ownership = orig_ownership;
+                if *pushed {
+                    sess.included_mod_stack.borrow_mut().pop();
+                }
                 result
             }
-
-            _ => noop_flat_map_item(item, self),
+            _ => {
+                item.attrs = attrs;
+                noop_flat_map_item(item, self)
+            }
         }
     }
 

src/librustc_expand/lib.rs

@@ -1,9 +1,11 @@
+#![feature(bool_to_option)]
 #![feature(cow_is_borrowed)]
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_span)]
+#![feature(try_blocks)]
 
 extern crate proc_macro as pm;
 
@@ -33,8 +35,10 @@ pub use mbe::macro_rules::compile_declarative_macro;
 crate use rustc_span::hygiene;
 pub mod base;
 pub mod build;
+#[macro_use]
+pub mod config;
 pub mod expand;
-pub use rustc_parse::config;
+pub mod module;
 pub mod proc_macro;
 
 crate mod mbe;