|
| 1 | +use rustc_hir::Mutability; |
| 2 | +use rustc_index::bit_set::HybridBitSet; |
| 3 | +use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; |
| 4 | +use rustc_middle::mir::{self, BasicBlock, Local, Location}; |
| 5 | +use rustc_middle::ty::TyCtxt; |
| 6 | + |
| 7 | +use crate::transform::{MirPass, MirSource}; |
| 8 | + |
| 9 | +/// This pass looks for MIR that always copies the same local into the return place and eliminates |
| 10 | +/// the copy by renaming all uses of that local to `_0`. |
| 11 | +/// |
| 12 | +/// This allows LLVM to perform an optimization similar to the named return value optimization |
| 13 | +/// (NRVO) that is guaranteed in C++. This avoids a stack allocation and `memcpy` for the |
| 14 | +/// relatively common pattern of allocating a buffer on the stack, mutating it, and returning it by |
| 15 | +/// value like so: |
| 16 | +/// |
| 17 | +/// ```rust |
| 18 | +/// fn foo(init: fn(&mut [u8; 1024])) -> [u8; 1024] { |
| 19 | +/// let mut buf = [0; 1024]; |
| 20 | +/// init(&mut buf); |
| 21 | +/// buf |
| 22 | +/// } |
| 23 | +/// ``` |
| 24 | +/// |
| 25 | +/// For now, this pass is very simple and only capable of eliminating a single copy. A more general |
| 26 | +/// version of copy propagation, such as the one based on non-overlapping live ranges in [#47954] and |
| 27 | +/// [#71003], could yield even more benefits. |
| 28 | +/// |
| 29 | +/// [#47954]: https://github.com/rust-lang/rust/pull/47954 |
| 30 | +/// [#71003]: https://github.com/rust-lang/rust/pull/71003 |
| 31 | +pub struct RenameReturnPlace; |
| 32 | + |
| 33 | +impl<'tcx> MirPass<'tcx> for RenameReturnPlace { |
| 34 | + fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut mir::Body<'tcx>) { |
| 35 | + if tcx.sess.opts.debugging_opts.mir_opt_level == 0 { |
| 36 | + return; |
| 37 | + } |
| 38 | + |
| 39 | + let returned_local = match local_eligible_for_nrvo(body) { |
| 40 | + Some(l) => l, |
| 41 | + None => { |
| 42 | + debug!("`{:?}` was ineligible for NRVO", src.def_id()); |
| 43 | + return; |
| 44 | + } |
| 45 | + }; |
| 46 | + |
| 47 | + // Sometimes, the return place is assigned a local of a different but coercable type, for |
| 48 | + // example `&T` instead of `&mut T`. Overwriting the `LocalInfo` for the return place would |
| 49 | + // result in it having an incorrect type. Although this doesn't seem to cause a problem in |
| 50 | + // codegen, bail out anyways since it happens so rarely. |
| 51 | + let ret_ty = body.local_decls[mir::RETURN_PLACE].ty; |
| 52 | + let assigned_ty = body.local_decls[returned_local].ty; |
| 53 | + if ret_ty != assigned_ty { |
| 54 | + debug!("`{:?}` was eligible for NRVO but for type mismatch", src.def_id()); |
| 55 | + debug!("typeof(_0) != typeof({:?}); {:?} != {:?}", returned_local, ret_ty, assigned_ty); |
| 56 | + return; |
| 57 | + } |
| 58 | + |
| 59 | + debug!( |
| 60 | + "`{:?}` was eligible for NRVO, making {:?} the return place", |
| 61 | + src.def_id(), |
| 62 | + returned_local |
| 63 | + ); |
| 64 | + |
| 65 | + RenameToReturnPlace { tcx, to_rename: returned_local }.visit_body(body); |
| 66 | + |
| 67 | + // Clean up the `NOP`s we inserted for statements made useless by our renaming. |
| 68 | + for block_data in body.basic_blocks_mut() { |
| 69 | + block_data.statements.retain(|stmt| stmt.kind != mir::StatementKind::Nop); |
| 70 | + } |
| 71 | + |
| 72 | + // Overwrite the debuginfo of `_0` with that of the renamed local. |
| 73 | + let (renamed_decl, ret_decl) = |
| 74 | + body.local_decls.pick2_mut(returned_local, mir::RETURN_PLACE); |
| 75 | + ret_decl.clone_from(renamed_decl); |
| 76 | + |
| 77 | + // The return place is always mutable. |
| 78 | + ret_decl.mutability = Mutability::Mut; |
| 79 | + } |
| 80 | +} |
| 81 | + |
| 82 | +/// MIR that is eligible for the NRVO must fulfill two conditions: |
| 83 | +/// 1. The return place must not be read prior to the `Return` terminator. |
| 84 | +/// 2. A simple assignment of a whole local to the return place (e.g., `_0 = _1`) must be the |
| 85 | +/// only definition of the return place reaching the `Return` terminator. |
| 86 | +/// |
| 87 | +/// If the MIR fulfills both these conditions, this function returns the `Local` that is assigned |
| 88 | +/// to the return place along all possible paths through the control-flow graph. |
| 89 | +fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> { |
| 90 | + if IsReturnPlaceRead::run(body) { |
| 91 | + return None; |
| 92 | + } |
| 93 | + |
| 94 | + let mut copied_to_return_place = None; |
| 95 | + for block in body.basic_blocks().indices() { |
| 96 | + // Look for blocks with a `Return` terminator. |
| 97 | + if !matches!(body[block].terminator().kind, mir::TerminatorKind::Return) { |
| 98 | + continue; |
| 99 | + } |
| 100 | + |
| 101 | + // Look for an assignment of a single local to the return place prior to the `Return`. |
| 102 | + let returned_local = find_local_assigned_to_return_place(block, body)?; |
| 103 | + match body.local_kind(returned_local) { |
| 104 | + // FIXME: Can we do this for arguments as well? |
| 105 | + mir::LocalKind::Arg => return None, |
| 106 | + |
| 107 | + mir::LocalKind::ReturnPointer => bug!("Return place was assigned to itself?"), |
| 108 | + mir::LocalKind::Var | mir::LocalKind::Temp => {} |
| 109 | + } |
| 110 | + |
| 111 | + // If multiple different locals are copied to the return place. We can't pick a |
| 112 | + // single one to rename. |
| 113 | + if copied_to_return_place.map_or(false, |old| old != returned_local) { |
| 114 | + return None; |
| 115 | + } |
| 116 | + |
| 117 | + copied_to_return_place = Some(returned_local); |
| 118 | + } |
| 119 | + |
| 120 | + return copied_to_return_place; |
| 121 | +} |
| 122 | + |
| 123 | +fn find_local_assigned_to_return_place( |
| 124 | + start: BasicBlock, |
| 125 | + body: &mut mir::Body<'_>, |
| 126 | +) -> Option<Local> { |
| 127 | + let mut block = start; |
| 128 | + let mut seen = HybridBitSet::new_empty(body.basic_blocks().len()); |
| 129 | + |
| 130 | + // Iterate as long as `block` has exactly one predecessor that we have not yet visited. |
| 131 | + while seen.insert(block) { |
| 132 | + trace!("Looking for assignments to `_0` in {:?}", block); |
| 133 | + |
| 134 | + let local = body[block].statements.iter().rev().find_map(as_local_assigned_to_return_place); |
| 135 | + if local.is_some() { |
| 136 | + return local; |
| 137 | + } |
| 138 | + |
| 139 | + match body.predecessors()[block].as_slice() { |
| 140 | + &[pred] => block = pred, |
| 141 | + _ => return None, |
| 142 | + } |
| 143 | + } |
| 144 | + |
| 145 | + return None; |
| 146 | +} |
| 147 | + |
| 148 | +// If this statement is an assignment of an unprojected local to the return place, |
| 149 | +// return that local. |
| 150 | +fn as_local_assigned_to_return_place(stmt: &mir::Statement<'_>) -> Option<Local> { |
| 151 | + if let mir::StatementKind::Assign(box (lhs, rhs)) = &stmt.kind { |
| 152 | + if lhs.as_local() == Some(mir::RETURN_PLACE) { |
| 153 | + if let mir::Rvalue::Use(mir::Operand::Copy(rhs) | mir::Operand::Move(rhs)) = rhs { |
| 154 | + return rhs.as_local(); |
| 155 | + } |
| 156 | + } |
| 157 | + } |
| 158 | + |
| 159 | + None |
| 160 | +} |
| 161 | + |
| 162 | +struct RenameToReturnPlace<'tcx> { |
| 163 | + to_rename: Local, |
| 164 | + tcx: TyCtxt<'tcx>, |
| 165 | +} |
| 166 | + |
| 167 | +/// Replaces all uses of `self.to_rename` with `_0`. |
| 168 | +impl MutVisitor<'tcx> for RenameToReturnPlace<'tcx> { |
| 169 | + fn tcx(&self) -> TyCtxt<'tcx> { |
| 170 | + self.tcx |
| 171 | + } |
| 172 | + |
| 173 | + fn visit_statement(&mut self, stmt: &mut mir::Statement<'tcx>, loc: Location) { |
| 174 | + // Remove assignments of the local being replaced to the return place, since it is now the |
| 175 | + // return place: |
| 176 | + // _0 = _1 |
| 177 | + if as_local_assigned_to_return_place(stmt) == Some(self.to_rename) { |
| 178 | + stmt.kind = mir::StatementKind::Nop; |
| 179 | + return; |
| 180 | + } |
| 181 | + |
| 182 | + // Remove storage annotations for the local being replaced: |
| 183 | + // StorageLive(_1) |
| 184 | + if let mir::StatementKind::StorageLive(local) | mir::StatementKind::StorageDead(local) = |
| 185 | + stmt.kind |
| 186 | + { |
| 187 | + if local == self.to_rename { |
| 188 | + stmt.kind = mir::StatementKind::Nop; |
| 189 | + return; |
| 190 | + } |
| 191 | + } |
| 192 | + |
| 193 | + self.super_statement(stmt, loc) |
| 194 | + } |
| 195 | + |
| 196 | + fn visit_terminator(&mut self, terminator: &mut mir::Terminator<'tcx>, loc: Location) { |
| 197 | + // Ignore the implicit "use" of the return place in a `Return` statement. |
| 198 | + if let mir::TerminatorKind::Return = terminator.kind { |
| 199 | + return; |
| 200 | + } |
| 201 | + |
| 202 | + self.super_terminator(terminator, loc); |
| 203 | + } |
| 204 | + |
| 205 | + fn visit_local(&mut self, l: &mut Local, _: PlaceContext, _: Location) { |
| 206 | + assert_ne!(*l, mir::RETURN_PLACE); |
| 207 | + if *l == self.to_rename { |
| 208 | + *l = mir::RETURN_PLACE; |
| 209 | + } |
| 210 | + } |
| 211 | +} |
| 212 | + |
| 213 | +struct IsReturnPlaceRead(bool); |
| 214 | + |
| 215 | +impl IsReturnPlaceRead { |
| 216 | + fn run(body: &mir::Body<'_>) -> bool { |
| 217 | + let mut vis = IsReturnPlaceRead(false); |
| 218 | + vis.visit_body(body); |
| 219 | + vis.0 |
| 220 | + } |
| 221 | +} |
| 222 | + |
| 223 | +impl Visitor<'tcx> for IsReturnPlaceRead { |
| 224 | + fn visit_local(&mut self, &l: &Local, ctxt: PlaceContext, _: Location) { |
| 225 | + if l == mir::RETURN_PLACE && ctxt.is_use() && !ctxt.is_place_assignment() { |
| 226 | + self.0 = true; |
| 227 | + } |
| 228 | + } |
| 229 | + |
| 230 | + fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, loc: Location) { |
| 231 | + // Ignore the implicit "use" of the return place in a `Return` statement. |
| 232 | + if let mir::TerminatorKind::Return = terminator.kind { |
| 233 | + return; |
| 234 | + } |
| 235 | + |
| 236 | + self.super_terminator(terminator, loc); |
| 237 | + } |
| 238 | +} |
0 commit comments