diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs index a66ddd27dbb2e..aafaace398530 100644 --- a/compiler/rustc_borrowck/src/type_check/input_output.rs +++ b/compiler/rustc_borrowck/src/type_check/input_output.rs @@ -110,28 +110,26 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } } + let body_yield_ty = body.yield_ty(self.tcx()); debug!( "equate_inputs_and_outputs: body.yield_ty {:?}, universal_regions.yield_ty {:?}", - body.yield_ty(), - universal_regions.yield_ty + body_yield_ty, universal_regions.yield_ty ); // We will not have a universal_regions.yield_ty if we yield (by accident) // outside of a generator and return an `impl Trait`, so emit a delay_span_bug // because we don't want to panic in an assert here if we've already got errors. - if body.yield_ty().is_some() != universal_regions.yield_ty.is_some() { + if body_yield_ty.is_some() != universal_regions.yield_ty.is_some() { self.tcx().sess.delay_span_bug( body.span, &format!( "Expected body to have yield_ty ({:?}) iff we have a UR yield_ty ({:?})", - body.yield_ty(), - universal_regions.yield_ty, + body_yield_ty, universal_regions.yield_ty, ), ); } - if let (Some(mir_yield_ty), Some(ur_yield_ty)) = - (body.yield_ty(), universal_regions.yield_ty) + if let (Some(mir_yield_ty), Some(ur_yield_ty)) = (body_yield_ty, universal_regions.yield_ty) { let yield_span = body.local_decls[RETURN_PLACE].source_info.span; self.equate_normalized_input_or_output(ur_yield_ty, mir_yield_ty, yield_span); diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 3c1c3ab45ce7a..cfc31b75685b4 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -1507,7 +1507,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { self.check_operand(value, term_location); let value_ty = value.ty(body, tcx); - match body.yield_ty() { + match body.yield_ty(tcx) { None => span_mirbug!(self, term, "yield in non-generator"), Some(ty) => { if let Err(terr) = self.sub_types( diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 163ccd9460c54..752fefd46c36c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1235,8 +1235,8 @@ fn generator_layout_and_saved_local_names<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, ) -> (&'tcx GeneratorLayout<'tcx>, IndexVec>) { + let generator_layout = &tcx.mir_generator_info(def_id).generator_layout; let body = tcx.optimized_mir(def_id); - let generator_layout = body.generator_layout().unwrap(); let mut generator_saved_local_names = IndexVec::from_elem(None, &generator_layout.field_tys); let state_arg = mir::Local::new(1); diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs index 22a61774e8cf7..a070c5d3f4816 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs @@ -225,7 +225,7 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> { // `async` functions cannot be `const fn`. This is checked during AST lowering, so there's // no need to emit duplicate errors here. - if self.ccx.is_async() || body.generator.is_some() { + if self.ccx.is_async() || tcx.generator_kind(def_id).is_some() { tcx.sess.delay_span_bug(body.span, "`async` functions cannot be `const fn`"); return; } diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs index f3ae16da43bd1..449b3d1aa0836 100644 --- a/compiler/rustc_const_eval/src/transform/promote_consts.rs +++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs @@ -964,7 +964,6 @@ pub fn promote_candidates<'tcx>( 0, vec![], body.span, - body.generator_kind(), body.tainted_by_errors, ); promoted.phase = MirPhase::Analysis(AnalysisPhase::Initial); diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index 81b82a21fa1a7..5187ae3beebde 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -65,6 +65,7 @@ impl<'tcx> MirPass<'tcx> for Validator { storage_liveness, place_cache: Vec::new(), value_cache: Vec::new(), + is_generator: tcx.generator_kind(def_id).is_some(), } .visit_body(body); } @@ -118,6 +119,7 @@ struct TypeChecker<'a, 'tcx> { storage_liveness: ResultsCursor<'a, 'tcx, MaybeStorageLive>, place_cache: Vec>, value_cache: Vec, + is_generator: bool, } impl<'a, 'tcx> TypeChecker<'a, 'tcx> { @@ -323,16 +325,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } &ty::Generator(def_id, substs, _) => { let f_ty = if let Some(var) = parent_ty.variant_index { - let gen_body = if def_id == self.body.source.def_id() { - self.body - } else { - self.tcx.optimized_mir(def_id) - }; - - let Some(layout) = gen_body.generator_layout() else { - self.fail(location, format!("No generator layout for {:?}", parent_ty)); - return; - }; + let generator_info = self.tcx.mir_generator_info(def_id); + let layout = &generator_info.generator_layout; let Some(&local) = layout.variant_fields[var].get(f) else { fail_out_of_bounds(self, location); @@ -861,7 +855,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } TerminatorKind::Yield { resume, drop, .. } => { - if self.body.generator.is_none() { + if !self.is_generator { self.fail(location, "`Yield` cannot appear outside generator bodies"); } if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { @@ -903,7 +897,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } TerminatorKind::GeneratorDrop => { - if self.body.generator.is_none() { + if !self.is_generator { self.fail(location, "`GeneratorDrop` cannot appear outside generator bodies"); } if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index a0a0855251b8d..9204ebd465998 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -202,6 +202,7 @@ provide! { tcx, def_id, other, cdata, thir_abstract_const => { table } optimized_mir => { table } mir_for_ctfe => { table } + mir_generator_info => { table } promoted_mir => { table } def_span => { table } def_ident_span => { table } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 87fa69e1639a1..666fd6dc76758 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -1426,6 +1426,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { debug!("EntryBuilder::encode_mir({:?})", def_id); if encode_opt { record!(self.tables.optimized_mir[def_id.to_def_id()] <- tcx.optimized_mir(def_id)); + + if let DefKind::Generator = self.tcx.def_kind(def_id) { + record!(self.tables.mir_generator_info[def_id.to_def_id()] <- tcx.mir_generator_info(def_id)); + } } if encode_const { record!(self.tables.mir_for_ctfe[def_id.to_def_id()] <- tcx.mir_for_ctfe(def_id)); diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 774cff2075d09..35a47018441a9 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -363,6 +363,7 @@ define_tables! { object_lifetime_default: Table>, optimized_mir: Table>>, mir_for_ctfe: Table>>, + mir_generator_info: Table>>, promoted_mir: Table>>>, // FIXME(compiler-errors): Why isn't this a LazyArray? thir_abstract_const: Table]>>, diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs index f8aae86fe3dc3..ae10a35cc5a71 100644 --- a/compiler/rustc_middle/src/arena.rs +++ b/compiler/rustc_middle/src/arena.rs @@ -13,6 +13,11 @@ macro_rules! arena_types { [] steal_thir: rustc_data_structures::steal::Steal>, [] steal_mir: rustc_data_structures::steal::Steal>, [decode] mir: rustc_middle::mir::Body<'tcx>, + [decode] generator_info: rustc_middle::mir::GeneratorInfo<'tcx>, + [] mir_generator_lowered: ( + rustc_data_structures::steal::Steal>, + Option>, + ), [] steal_promoted: rustc_data_structures::steal::Steal< rustc_index::vec::IndexVec< diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 068daaadbdac2..c869b14ed1135 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -203,18 +203,11 @@ impl<'tcx> MirSource<'tcx> { #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable, TypeFoldable, TypeVisitable)] pub struct GeneratorInfo<'tcx> { - /// The yield type of the function, if it is a generator. - pub yield_ty: Option>, - /// Generator drop glue. - pub generator_drop: Option>, + pub generator_drop: Body<'tcx>, /// The layout of a generator. Produced by the state transformation. - pub generator_layout: Option>, - - /// If this is a generator then record the type of source expression that caused this generator - /// to be created. - pub generator_kind: GeneratorKind, + pub generator_layout: GeneratorLayout<'tcx>, } /// The lowered representation of a single function. @@ -240,8 +233,6 @@ pub struct Body<'tcx> { /// and used for debuginfo. Indexed by a `SourceScope`. pub source_scopes: IndexVec>, - pub generator: Option>>, - /// Declarations of locals. /// /// The first local is the return value pointer, followed by `arg_count` @@ -306,7 +297,6 @@ impl<'tcx> Body<'tcx> { arg_count: usize, var_debug_info: Vec>, span: Span, - generator_kind: Option, tainted_by_errors: Option, ) -> Self { // We need `arg_count` locals, and one for the return place. @@ -323,14 +313,6 @@ impl<'tcx> Body<'tcx> { source, basic_blocks: BasicBlocks::new(basic_blocks), source_scopes, - generator: generator_kind.map(|generator_kind| { - Box::new(GeneratorInfo { - yield_ty: None, - generator_drop: None, - generator_layout: None, - generator_kind, - }) - }), local_decls, user_type_annotations, arg_count, @@ -357,7 +339,6 @@ impl<'tcx> Body<'tcx> { source: MirSource::item(CRATE_DEF_ID.to_def_id()), basic_blocks: BasicBlocks::new(basic_blocks), source_scopes: IndexVec::new(), - generator: None, local_decls: IndexVec::new(), user_type_annotations: IndexVec::new(), arg_count: 0, @@ -489,24 +470,15 @@ impl<'tcx> Body<'tcx> { .unwrap_or_else(|| Either::Right(block_data.terminator())) } - #[inline] - pub fn yield_ty(&self) -> Option> { - self.generator.as_ref().and_then(|generator| generator.yield_ty) - } - - #[inline] - pub fn generator_layout(&self) -> Option<&GeneratorLayout<'tcx>> { - self.generator.as_ref().and_then(|generator| generator.generator_layout.as_ref()) - } - - #[inline] - pub fn generator_drop(&self) -> Option<&Body<'tcx>> { - self.generator.as_ref().and_then(|generator| generator.generator_drop.as_ref()) - } - - #[inline] - pub fn generator_kind(&self) -> Option { - self.generator.as_ref().map(|generator| generator.generator_kind) + pub fn yield_ty(&self, tcx: TyCtxt<'_>) -> Option> { + if tcx.generator_kind(self.source.def_id()).is_none() { + return None; + }; + let gen_ty = self.local_decls.raw[1].ty; + match *gen_ty.kind() { + ty::Generator(_, substs, _) => Some(substs.as_generator().sig().yield_ty), + _ => None, + } } } diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 05dcfba77b233..50e5f79b15f51 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -131,9 +131,6 @@ fn dump_matched_mir_node<'tcx, F>( Some(promoted) => write!(file, "::{:?}`", promoted)?, } writeln!(file, " {} {}", disambiguator, pass_name)?; - if let Some(ref layout) = body.generator_layout() { - writeln!(file, "/* generator_layout = {:#?} */", layout)?; - } writeln!(file)?; extra_data(PassWhere::BeforeCFG, &mut file)?; write_user_type_annotations(tcx, body, &mut file)?; @@ -1012,7 +1009,7 @@ fn write_mir_sig(tcx: TyCtxt<'_>, body: &Body<'_>, w: &mut dyn Write) -> io::Res write!(w, ": {} =", body.return_ty())?; } - if let Some(yield_ty) = body.yield_ty() { + if let Some(yield_ty) = body.yield_ty(tcx) { writeln!(w)?; writeln!(w, "yields {}", yield_ty)?; } diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index ddcf3711bfc95..097ca04f65a64 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -970,16 +970,6 @@ macro_rules! extra_body_methods { macro_rules! super_body { ($self:ident, $body:ident, $($mutability:ident, $invalidate:tt)?) => { - let span = $body.span; - if let Some(gen) = &$($mutability)? $body.generator { - if let Some(yield_ty) = $(& $mutability)? gen.yield_ty { - $self.visit_ty( - yield_ty, - TyContext::YieldTy(SourceInfo::outermost(span)) - ); - } - } - for (bb, data) in basic_blocks_iter!($body, $($mutability, $invalidate)?) { $self.visit_basic_block_data(bb, data); } diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index a098e570305c8..991874b9066f0 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -419,9 +419,10 @@ rustc_queries! { } } - query mir_drops_elaborated_and_const_checked( - key: ty::WithOptConstParam - ) -> &'tcx Steal> { + query mir_drops_elaborated_and_const_checked(key: ty::WithOptConstParam) -> &'tcx ( + Steal>, + Option>, + ) { no_hash desc { |tcx| "elaborating drops for `{}`", tcx.def_path_str(key.did.to_def_id()) } } @@ -465,6 +466,12 @@ rustc_queries! { } } + query mir_generator_info(key: DefId) -> &'tcx mir::GeneratorInfo<'tcx> { + desc { |tcx| "generator glue MIR for `{}`", tcx.def_path_str(key) } + cache_on_disk_if { key.is_local() } + separate_provide_extern + } + /// MIR after our optimization passes have run. This is MIR that is ready /// for codegen. This is also the only query that can fetch non-local MIR, at present. query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> { diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs index 14ec88b7e0d7e..f9ec5ada09fd5 100644 --- a/compiler/rustc_middle/src/ty/codec.rs +++ b/compiler/rustc_middle/src/ty/codec.rs @@ -381,6 +381,7 @@ impl_decodable_via_ref! { &'tcx ty::List>>, &'tcx traits::ImplSource<'tcx, ()>, &'tcx mir::Body<'tcx>, + &'tcx mir::GeneratorInfo<'tcx>, &'tcx mir::UnsafetyCheckResult, &'tcx mir::BorrowCheckResult<'tcx>, &'tcx mir::coverage::CodeRegion, diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index a42d05706137c..26bd01fd578a3 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -2509,7 +2509,11 @@ impl<'tcx> TyCtxt<'tcx> { /// Returns layout of a generator. Layout might be unavailable if the /// generator is tainted by errors. pub fn generator_layout(self, def_id: DefId) -> Option<&'tcx GeneratorLayout<'tcx>> { - self.optimized_mir(def_id).generator_layout() + if self.generator_kind(def_id).is_some() { + Some(&self.mir_generator_info(def_id).generator_layout) + } else { + None + } } /// Given the `DefId` of an impl, returns the `DefId` of the trait it implements. diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs index e1e705a922f9e..07bdfeda124cd 100644 --- a/compiler/rustc_middle/src/ty/parameterized.rs +++ b/compiler/rustc_middle/src/ty/parameterized.rs @@ -3,7 +3,7 @@ use rustc_hir::def_id::{DefId, DefIndex}; use rustc_index::vec::{Idx, IndexVec}; use crate::middle::exported_symbols::ExportedSymbol; -use crate::mir::Body; +use crate::mir::{Body, GeneratorInfo}; use crate::ty::abstract_const::Node; use crate::ty::{ self, Const, FnSig, GeneratorDiagnosticData, GenericPredicates, Predicate, TraitRef, Ty, @@ -124,6 +124,7 @@ parameterized_over_tcx! { Predicate, GeneratorDiagnosticData, Body, + GeneratorInfo, Node, ExportedSymbol, } diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs index cbcf9cd129f3f..a4e2fcbd124f7 100644 --- a/compiler/rustc_mir_build/src/build/mod.rs +++ b/compiler/rustc_mir_build/src/build/mod.rs @@ -88,8 +88,7 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) -> Body<'_ debug_assert!( !(body.local_decls.has_free_regions() || body.basic_blocks.has_free_regions() - || body.var_debug_info.has_free_regions() - || body.yield_ty().has_free_regions()), + || body.var_debug_info.has_free_regions()), "Unexpected free regions in MIR: {:?}", body, ); @@ -468,7 +467,7 @@ fn construct_fn<'tcx>( let arguments = &thir.params; - let (yield_ty, return_ty) = if generator_kind.is_some() { + let return_ty = if generator_kind.is_some() { let gen_ty = arguments[thir::UPVAR_ENV_PARAM].ty; let gen_sig = match gen_ty.kind() { ty::Generator(_, gen_substs, ..) => gen_substs.as_generator().sig(), @@ -476,9 +475,9 @@ fn construct_fn<'tcx>( span_bug!(span, "generator w/o generator type: {:?}", gen_ty) } }; - (Some(gen_sig.yield_ty), gen_sig.return_ty) + gen_sig.return_ty } else { - (None, fn_sig.output()) + fn_sig.output() }; let infcx = tcx.infer_ctxt().build(); @@ -531,9 +530,6 @@ fn construct_fn<'tcx>( } else { None }; - if yield_ty.is_some() { - body.generator.as_mut().unwrap().yield_ty = yield_ty; - } body } @@ -616,7 +612,6 @@ fn construct_error<'tcx>( ) -> Body<'tcx> { let span = tcx.def_span(def); let hir_id = tcx.hir().local_def_id_to_hir_id(def); - let generator_kind = tcx.generator_kind(def); let ty = tcx.ty_error(); let num_params = match body_owner_kind { @@ -658,7 +653,7 @@ fn construct_error<'tcx>( } cfg.terminate(START_BLOCK, source_info, TerminatorKind::Unreachable); - let mut body = Body::new( + Body::new( MirSource::item(def.to_def_id()), cfg.basic_blocks, source_scopes, @@ -667,11 +662,8 @@ fn construct_error<'tcx>( num_params, vec![], span, - generator_kind, Some(err), - ); - body.generator.as_mut().map(|gen| gen.yield_ty = Some(ty)); - body + ) } impl<'a, 'tcx> Builder<'a, 'tcx> { @@ -758,7 +750,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { self.arg_count, self.var_debug_info, self.fn_span, - self.generator_kind, self.typeck_results.tainted_by_errors, ) } diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 4e4515888454b..be443a7201979 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -138,7 +138,6 @@ impl<'tcx> MirPass<'tcx> for ConstProp { body.arg_count, Default::default(), body.span, - body.generator_kind(), body.tainted_by_errors, ); diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 479c4e577d4e3..623cedc78f903 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -111,7 +111,6 @@ impl<'tcx> MirLint<'tcx> for ConstProp { body.arg_count, Default::default(), body.span, - body.generator_kind(), body.tainted_by_errors, ); diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index c833de3a8a750..45569da0a8067 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -52,7 +52,6 @@ use crate::deref_separator::deref_finder; use crate::simplify; use crate::util::expand_aggregate; -use crate::MirPass; use rustc_data_structures::fx::FxHashMap; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; @@ -72,8 +71,6 @@ use rustc_target::abi::VariantIdx; use rustc_target::spec::PanicStrategy; use std::{iter, ops}; -pub struct StateTransform; - struct RenameLocalVisitor<'tcx> { from: Local, to: Local, @@ -971,8 +968,6 @@ fn create_generator_drop_shim<'tcx>( // unrelated code from the resume part of the function simplify::remove_dead_blocks(tcx, &mut body); - dump_mir(tcx, None, "generator_drop", &0, &body, |_, _| Ok(())); - body } @@ -1072,6 +1067,7 @@ fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool { fn create_generator_resume_function<'tcx>( tcx: TyCtxt<'tcx>, + generator_kind: hir::GeneratorKind, transform: TransformVisitor<'tcx>, body: &mut Body<'tcx>, can_return: bool, @@ -1117,8 +1113,6 @@ fn create_generator_resume_function<'tcx>( cases.insert(0, (UNRESUMED, BasicBlock::new(0))); // Panic when resumed on the returned or poisoned state - let generator_kind = body.generator_kind().unwrap(); - if can_unwind { cases.insert( 1, @@ -1141,8 +1135,6 @@ fn create_generator_resume_function<'tcx>( // Make sure we remove dead blocks to remove // unrelated code from the drop part of the function simplify::remove_dead_blocks(tcx, body); - - dump_mir(tcx, None, "generator_resume", &0, body, |_, _| Ok(())); } fn insert_clean_drop(body: &mut Body<'_>) -> BasicBlock { @@ -1238,141 +1230,146 @@ fn create_cases<'tcx>( .collect() } -impl<'tcx> MirPass<'tcx> for StateTransform { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let Some(yield_ty) = body.yield_ty() else { - // This only applies to generators - return; - }; +pub(crate) fn state_transform_body_for_generator<'tcx>( + tcx: TyCtxt<'tcx>, + generator_kind: hir::GeneratorKind, + body: &mut Body<'tcx>, +) -> GeneratorInfo<'tcx> { + // The first argument is the generator type passed by value + let gen_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; + + // Get the interior types and substs which typeck computed + let (yield_ty, return_ty, upvars, interior, discr_ty, movable) = match *gen_ty.kind() { + ty::Generator(_, substs, movability) => { + let substs = substs.as_generator(); + ( + substs.sig().yield_ty, + substs.sig().return_ty, + substs.upvar_tys().collect(), + substs.witness(), + substs.discr_ty(tcx), + movability == hir::Movability::Movable, + ) + } + _ => span_bug!(body.span, "unexpected generator type {}", gen_ty), + }; - assert!(body.generator_drop().is_none()); - - // The first argument is the generator type passed by value - let gen_ty = body.local_decls.raw[1].ty; - - // Get the interior types and substs which typeck computed - let (upvars, interior, discr_ty, movable) = match *gen_ty.kind() { - ty::Generator(_, substs, movability) => { - let substs = substs.as_generator(); - ( - substs.upvar_tys().collect(), - substs.witness(), - substs.discr_ty(tcx), - movability == hir::Movability::Movable, - ) - } - _ => { - tcx.sess - .delay_span_bug(body.span, &format!("unexpected generator type {}", gen_ty)); - return; - } - }; + // Compute GeneratorState + let state_did = tcx.require_lang_item(LangItem::GeneratorState, None); + let state_adt_ref = tcx.adt_def(state_did); + let state_substs = tcx.intern_substs(&[yield_ty.into(), return_ty.into()]); + let ret_ty = tcx.mk_adt(state_adt_ref, state_substs); + + // We rename RETURN_PLACE which has type mir.return_ty to new_ret_local + // RETURN_PLACE then is a fresh unused local with type ret_ty. + let new_ret_local = replace_local(RETURN_PLACE, ret_ty, body, tcx); + + // We also replace the resume argument and insert an `Assign`. + // This is needed because the resume argument `_2` might be live across a `yield`, in which + // case there is no `Assign` to it that the transform can turn into a store to the generator + // state. After the yield the slot in the generator state would then be uninitialized. + let resume_local = Local::new(2); + let new_resume_local = + replace_local(resume_local, body.local_decls[resume_local].ty, body, tcx); + + // When first entering the generator, move the resume argument into its new local. + let source_info = SourceInfo::outermost(body.span); + let stmts = &mut body.basic_blocks_mut()[BasicBlock::new(0)].statements; + stmts.insert( + 0, + Statement { + source_info, + kind: StatementKind::Assign(Box::new(( + new_resume_local.into(), + Rvalue::Use(Operand::Move(resume_local.into())), + ))), + }, + ); - // Compute GeneratorState - let state_did = tcx.require_lang_item(LangItem::GeneratorState, None); - let state_adt_ref = tcx.adt_def(state_did); - let state_substs = tcx.intern_substs(&[yield_ty.into(), body.return_ty().into()]); - let ret_ty = tcx.mk_adt(state_adt_ref, state_substs); - - // We rename RETURN_PLACE which has type mir.return_ty to new_ret_local - // RETURN_PLACE then is a fresh unused local with type ret_ty. - let new_ret_local = replace_local(RETURN_PLACE, ret_ty, body, tcx); - - // We also replace the resume argument and insert an `Assign`. - // This is needed because the resume argument `_2` might be live across a `yield`, in which - // case there is no `Assign` to it that the transform can turn into a store to the generator - // state. After the yield the slot in the generator state would then be uninitialized. - let resume_local = Local::new(2); - let new_resume_local = - replace_local(resume_local, body.local_decls[resume_local].ty, body, tcx); - - // When first entering the generator, move the resume argument into its new local. - let source_info = SourceInfo::outermost(body.span); - let stmts = &mut body.basic_blocks_mut()[BasicBlock::new(0)].statements; - stmts.insert( - 0, - Statement { - source_info, - kind: StatementKind::Assign(Box::new(( - new_resume_local.into(), - Rvalue::Use(Operand::Move(resume_local.into())), - ))), - }, - ); + let always_live_locals = always_storage_live_locals(&body); - let always_live_locals = always_storage_live_locals(&body); + let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); - let liveness_info = - locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + sanitize_witness(tcx, body, interior, upvars, &liveness_info.saved_locals); - sanitize_witness(tcx, body, interior, upvars, &liveness_info.saved_locals); + if tcx.sess.opts.unstable_opts.validate_mir { + let mut vis = EnsureGeneratorFieldAssignmentsNeverAlias { + assigned_local: None, + saved_locals: &liveness_info.saved_locals, + storage_conflicts: &liveness_info.storage_conflicts, + }; - if tcx.sess.opts.unstable_opts.validate_mir { - let mut vis = EnsureGeneratorFieldAssignmentsNeverAlias { - assigned_local: None, - saved_locals: &liveness_info.saved_locals, - storage_conflicts: &liveness_info.storage_conflicts, - }; + vis.visit_body(body); + } - vis.visit_body(body); - } + // Extract locals which are live across suspension point into `layout` + // `remap` gives a mapping from local indices onto generator struct indices + // `storage_liveness` tells us which locals have live storage at suspension points + let (remap, generator_layout, storage_liveness) = compute_layout(liveness_info, body); + + let can_return = can_return(tcx, body, tcx.param_env(body.source.def_id())); + + // Run the transformation which converts Places from Local to generator struct + // accesses for locals in `remap`. + // It also rewrites `return x` and `yield y` as writing a new generator state and returning + // GeneratorState::Complete(x) and GeneratorState::Yielded(y) respectively. + let mut transform = TransformVisitor { + tcx, + state_adt_ref, + state_substs, + remap, + storage_liveness, + always_live_locals, + suspension_points: Vec::new(), + new_ret_local, + discr_ty, + }; + transform.visit_body(body); - // Extract locals which are live across suspension point into `layout` - // `remap` gives a mapping from local indices onto generator struct indices - // `storage_liveness` tells us which locals have live storage at suspension points - let (remap, layout, storage_liveness) = compute_layout(liveness_info, body); - - let can_return = can_return(tcx, body, tcx.param_env(body.source.def_id())); - - // Run the transformation which converts Places from Local to generator struct - // accesses for locals in `remap`. - // It also rewrites `return x` and `yield y` as writing a new generator state and returning - // GeneratorState::Complete(x) and GeneratorState::Yielded(y) respectively. - let mut transform = TransformVisitor { - tcx, - state_adt_ref, - state_substs, - remap, - storage_liveness, - always_live_locals, - suspension_points: Vec::new(), - new_ret_local, - discr_ty, - }; - transform.visit_body(body); + // Update our MIR struct to reflect the changes we've made + body.arg_count = 2; // self, resume arg + body.spread_arg = None; - // Update our MIR struct to reflect the changes we've made - body.arg_count = 2; // self, resume arg - body.spread_arg = None; + // Insert `drop(generator_struct)` which is used to drop upvars for generators in + // the unresumed state. + // This is expanded to a drop ladder in `elaborate_generator_drops`. + let drop_clean = insert_clean_drop(body); - body.generator.as_mut().unwrap().yield_ty = None; - body.generator.as_mut().unwrap().generator_layout = Some(layout); + dump_mir(tcx, None, "generator_pre-elab", &0, body, |_, _| Ok(())); - // Insert `drop(generator_struct)` which is used to drop upvars for generators in - // the unresumed state. - // This is expanded to a drop ladder in `elaborate_generator_drops`. - let drop_clean = insert_clean_drop(body); + // Expand `drop(generator_struct)` to a drop ladder which destroys upvars. + // If any upvars are moved out of, drop elaboration will handle upvar destruction. + // However we need to also elaborate the code generated by `insert_clean_drop`. + elaborate_generator_drops(tcx, body); - dump_mir(tcx, None, "generator_pre-elab", &0, body, |_, _| Ok(())); + dump_mir(tcx, None, "generator_post-transform", &0, body, |_, _| Ok(())); - // Expand `drop(generator_struct)` to a drop ladder which destroys upvars. - // If any upvars are moved out of, drop elaboration will handle upvar destruction. - // However we need to also elaborate the code generated by `insert_clean_drop`. - elaborate_generator_drops(tcx, body); + // Create a copy of our MIR and use it to create the drop shim for the generator + let generator_drop = create_generator_drop_shim(tcx, &transform, gen_ty, body, drop_clean); - dump_mir(tcx, None, "generator_post-transform", &0, body, |_, _| Ok(())); + dump_mir(tcx, None, "generator_drop", &0, &generator_drop, |position, file| { + if let PassWhere::BeforeCFG = position { + writeln!(file, "/* generator_layout = {:#?} */", generator_layout)?; + writeln!(file)?; + } + Ok(()) + }); - // Create a copy of our MIR and use it to create the drop shim for the generator - let drop_shim = create_generator_drop_shim(tcx, &transform, gen_ty, body, drop_clean); + // Create the Generator::resume function + create_generator_resume_function(tcx, generator_kind, transform, body, can_return); - body.generator.as_mut().unwrap().generator_drop = Some(drop_shim); + dump_mir(tcx, None, "generator_resume", &0, body, |position, file| { + if let PassWhere::BeforeCFG = position { + writeln!(file, "/* generator_layout = {:#?} */", generator_layout)?; + writeln!(file)?; + } + Ok(()) + }); - // Create the Generator::resume function - create_generator_resume_function(tcx, transform, body, can_return); + // Run derefer to fix Derefs that are not in the first place + deref_finder(tcx, body); - // Run derefer to fix Derefs that are not in the first place - deref_finder(tcx, body); - } + GeneratorInfo { generator_drop, generator_layout } } /// Looks for any assignments between locals (e.g., `_4 = _5`) that will both be converted to fields diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 780b91d9215d5..85de49776e41f 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -81,7 +81,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { // Avoid inlining into generators, since their `optimized_mir` is used for layout computation, // which can create a cycle, even when no attempt is made to inline the function in the other // direction. - if body.generator.is_some() { + if tcx.generator_kind(def_id).is_some() { return false; } @@ -887,16 +887,8 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } &ty::Generator(def_id, substs, _) => { let f_ty = if let Some(var) = parent_ty.variant_index { - let gen_body = if def_id == self.callee_body.source.def_id() { - self.callee_body - } else { - self.tcx.optimized_mir(def_id) - }; - - let Some(layout) = gen_body.generator_layout() else { - self.validation = Err("malformed MIR"); - return; - }; + let generator_info = self.tcx.mir_generator_info(def_id); + let layout = &generator_info.generator_layout; let Some(&local) = layout.variant_fields[var].get(f) else { self.validation = Err("malformed MIR"); diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 4791be1306c1b..de78395b7c637 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -27,9 +27,9 @@ use rustc_hir::intravisit::{self, Visitor}; use rustc_index::vec::IndexVec; use rustc_middle::mir::visit::Visitor as _; use rustc_middle::mir::{ - traversal, AnalysisPhase, Body, ConstQualifs, Constant, LocalDecl, MirPass, MirPhase, Operand, - Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, Statement, StatementKind, - TerminatorKind, + traversal, AnalysisPhase, Body, ConstQualifs, Constant, GeneratorInfo, LocalDecl, MirPass, + MirPhase, Operand, PassWhere, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, + SourceInfo, Statement, StatementKind, TerminatorKind, }; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt, TypeVisitable}; @@ -123,6 +123,7 @@ pub fn provide(providers: &mut Providers) { mir_drops_elaborated_and_const_checked, mir_for_ctfe, mir_for_ctfe_of_const_arg, + mir_generator_info, optimized_mir, is_mir_available, is_ctfe_mir_available: |tcx, did| is_mir_available(tcx, did), @@ -389,7 +390,7 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) - .body_const_context(def.did) .expect("mir_for_ctfe should not be used for runtime functions"); - let body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone(); + let body = tcx.mir_drops_elaborated_and_const_checked(def).0.borrow().clone(); let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::Const); @@ -427,7 +428,7 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) - fn mir_drops_elaborated_and_const_checked<'tcx>( tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam, -) -> &'tcx Steal> { +) -> &'tcx (Steal>, Option>) { if let Some(def) = def.try_upgrade(tcx) { return tcx.mir_drops_elaborated_and_const_checked(def); } @@ -451,12 +452,16 @@ fn mir_drops_elaborated_and_const_checked<'tcx>( body.tainted_by_errors = Some(error_reported); } - run_analysis_to_runtime_passes(tcx, &mut body); + let generator_layout = run_analysis_to_runtime_passes(tcx, true, &mut body); - tcx.alloc_steal_mir(body) + tcx.arena.alloc((Steal::new(body), generator_layout)) } -fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { +fn run_analysis_to_runtime_passes<'tcx>( + tcx: TyCtxt<'tcx>, + lower_generator: bool, + body: &mut Body<'tcx>, +) -> Option> { assert!(body.phase == MirPhase::Analysis(AnalysisPhase::Initial)); let did = body.source.def_id(); @@ -479,12 +484,14 @@ fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx> } debug!("runtime_mir_lowering({:?})", did); - run_runtime_lowering_passes(tcx, body); + let generator_layout = run_runtime_lowering_passes(tcx, lower_generator, body); assert!(body.phase == MirPhase::Runtime(RuntimePhase::Initial)); debug!("runtime_mir_cleanup({:?})", did); run_runtime_cleanup_passes(tcx, body); assert!(body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup)); + + generator_layout } // FIXME(JakobDegen): Can we make these lists of passes consts? @@ -504,8 +511,12 @@ fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { } /// Returns the sequence of passes that lowers analysis to runtime MIR. -fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let passes: &[&dyn MirPass<'tcx>] = &[ +fn run_runtime_lowering_passes<'tcx>( + tcx: TyCtxt<'tcx>, + lower_generator: bool, + body: &mut Body<'tcx>, +) -> Option> { + let pre_generator_passes: &[&dyn MirPass<'tcx>] = &[ // These next passes must be executed together &add_call_guards::CriticalCallEdges, &elaborate_drops::ElaborateDrops, @@ -519,14 +530,45 @@ fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // `AddRetag` needs to run after `ElaborateDrops`. Otherwise it should run fairly late, // but before optimizations begin. &elaborate_box_derefs::ElaborateBoxDerefs, - &generator::StateTransform, + ]; + let post_generator_passes: &[&dyn MirPass<'tcx>] = &[ &add_retag::AddRetag, // Deaggregator is necessary for const prop. We may want to consider implementing // CTFE support for aggregates. &deaggregator::Deaggregator, &Lint(const_prop_lint::ConstProp), ]; - pm::run_passes_no_validate(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::Initial))); + pm::run_passes_no_validate(tcx, body, pre_generator_passes, None); + + // We cannot use the pass manager for this transform, as it does not support additional output + // values, and we do not want to disable this pass. + rustc_middle::mir::dump_mir(tcx, None, "StateTransform", &"before", &body, |_, _| Ok(())); + let generator_layout = if lower_generator + && let Some(generator_kind) = tcx.generator_kind(body.source.def_id()) + { + Some(generator::state_transform_body_for_generator(tcx, generator_kind, body)) + } else { + None + }; + // Output generator layout once, so we don't have to do it on each dump. + rustc_middle::mir::dump_mir(tcx, None, "StateTransform", &"after", &body, |position, file| { + if let PassWhere::BeforeCFG = position + && let Some(ref generator_layout) = generator_layout + { + writeln!(file, "/* generator_layout = {:#?} */", generator_layout.generator_layout)?; + writeln!(file)?; + } + Ok(()) + }); + + pm::run_passes_no_validate( + tcx, + body, + post_generator_passes, + Some(MirPhase::Runtime(RuntimePhase::Initial)), + ); + + generator_layout } /// Returns the sequence of passes that do the initial cleanup of runtime MIR. @@ -537,7 +579,22 @@ fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &simplify::SimplifyCfg::new("elaborate-drops"), ]; - pm::run_passes(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::PostCleanup))); + // Do not validate, as this would cause a query cycle with `mir_generator_info`. + pm::run_passes_no_validate( + tcx, + body, + passes, + Some(MirPhase::Runtime(RuntimePhase::PostCleanup)), + ); +} + +fn mir_generator_info<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx GeneratorInfo<'tcx> { + let def_id = def_id.expect_local(); + let (_body, generator_layout) = + tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(def_id)); + generator_layout + .as_ref() + .unwrap_or_else(|| bug!("mir_generator_info called for non-generator {:?}", def_id)) } fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -621,7 +678,7 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { } debug!("about to call mir_drops_elaborated..."); let body = - tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal(); + tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).0.steal(); let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::NotConst); debug!("body: {:#?}", body); run_optimization_passes(tcx, &mut body); @@ -648,7 +705,8 @@ fn promoted_mir<'tcx>( if let Some(error_reported) = tainted_by_errors { body.tainted_by_errors = Some(error_reported); } - run_analysis_to_runtime_passes(tcx, body); + // Do not lower generators for promoted. + run_analysis_to_runtime_passes(tcx, false, body); } debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs index abe6cb285f505..88be81a2c60df 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/reveal_all.rs @@ -14,10 +14,6 @@ impl<'tcx> MirPass<'tcx> for RevealAll { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // Do not apply this transformation to generators. - if body.generator.is_some() { - return; - } - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); RevealAllVisitor { tcx, param_env }.visit_body_preserves_cfg(body); } diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index bf5906741441d..4208dc1cfa352 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -69,7 +69,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' // FIXME(#91576): Drop shims for generators aren't subject to the MIR passes at the end // of this function. Is this intentional? if let Some(ty::Generator(gen_def_id, substs, _)) = ty.map(Ty::kind) { - let body = tcx.optimized_mir(*gen_def_id).generator_drop().unwrap(); + let body = &tcx.mir_generator_info(*gen_def_id).generator_drop; let body = EarlyBinder(body.clone()).subst(tcx, substs); debug!("make_shim({:?}) = {:?}", instance, body); return body; @@ -235,7 +235,6 @@ fn new_body<'tcx>( arg_count, vec![], span, - None, // FIXME(compiler-errors): is this correct? None, ) diff --git a/src/test/mir-opt/generator_drop_cleanup.main-{closure#0}.generator_drop.0.mir b/src/test/mir-opt/generator_drop_cleanup.main-{closure#0}.generator_drop.0.mir index c718138b6b37e..f049b964f0f42 100644 --- a/src/test/mir-opt/generator_drop_cleanup.main-{closure#0}.generator_drop.0.mir +++ b/src/test/mir-opt/generator_drop_cleanup.main-{closure#0}.generator_drop.0.mir @@ -1,4 +1,5 @@ // MIR for `main::{closure#0}` 0 generator_drop + /* generator_layout = GeneratorLayout { field_tys: { _0: std::string::String, diff --git a/src/test/mir-opt/generator_tiny.main-{closure#0}.generator_resume.0.mir b/src/test/mir-opt/generator_tiny.main-{closure#0}.generator_resume.0.mir index 07aeeaae012c9..dbca8ddd46e2c 100644 --- a/src/test/mir-opt/generator_tiny.main-{closure#0}.generator_resume.0.mir +++ b/src/test/mir-opt/generator_tiny.main-{closure#0}.generator_resume.0.mir @@ -1,4 +1,5 @@ // MIR for `main::{closure#0}` 0 generator_resume + /* generator_layout = GeneratorLayout { field_tys: { _0: HasDrop,