From aede8f5fbff92bd1b257685c2e654a9bdf3a021f Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sun, 9 Feb 2025 01:57:23 -0800 Subject: [PATCH 1/4] Simplify `slice::Iter::next` enough that it inlines --- library/core/src/slice/iter.rs | 2 +- library/core/src/slice/iter/macros.rs | 22 +- tests/codegen/slice-iter-nonnull.rs | 6 +- ...ated_loop.PreCodegen.after.panic-abort.mir | 255 +++++++++++++----- ...ward_loop.PreCodegen.after.panic-abort.mir | 190 ++++++++++--- ...ard_loop.PreCodegen.after.panic-unwind.mir | 198 +++++++++++--- ...iter_next.PreCodegen.after.panic-abort.mir | 124 ++++++++- ...ter_next.PreCodegen.after.panic-unwind.mir | 124 ++++++++- 8 files changed, 771 insertions(+), 150 deletions(-) diff --git a/library/core/src/slice/iter.rs b/library/core/src/slice/iter.rs index a687ed7129dc8..5a87f1327dd74 100644 --- a/library/core/src/slice/iter.rs +++ b/library/core/src/slice/iter.rs @@ -11,7 +11,7 @@ use crate::iter::{ use crate::marker::PhantomData; use crate::mem::{self, SizedTypeProperties}; use crate::num::NonZero; -use crate::ptr::{NonNull, without_provenance, without_provenance_mut}; +use crate::ptr::{NonNull, null, without_provenance, without_provenance_mut}; use crate::{cmp, fmt}; #[stable(feature = "boxed_slice_into_iter", since = "1.80.0")] diff --git a/library/core/src/slice/iter/macros.rs b/library/core/src/slice/iter/macros.rs index 45e320e66bc1a..74dfb8634f2ff 100644 --- a/library/core/src/slice/iter/macros.rs +++ b/library/core/src/slice/iter/macros.rs @@ -154,16 +154,26 @@ macro_rules! iterator { #[inline] fn next(&mut self) -> Option<$elem> { - // could be implemented with slices, but this avoids bounds checks + // intentionally not using the helpers because this is + // one of the most mono'd things in the library. - // SAFETY: The call to `next_unchecked` is - // safe since we check if the iterator is empty first. + let ptr = self.ptr; + let end_or_len = self.end_or_len; + // SAFETY: Type invariants. unsafe { - if is_empty!(self) { - None + if T::IS_ZST { + let byte_end = end_or_len as *const u8; + if byte_end == null() { + return None; + } + self.end_or_len = byte_end.wrapping_sub(1) as _; } else { - Some(self.next_unchecked()) + if ptr == crate::intrinsics::transmute::<*const T, NonNull>(end_or_len) { + return None; + } + self.ptr = ptr.add(1); } + crate::intrinsics::transmute::, Option<$elem>>(ptr) } } diff --git a/tests/codegen/slice-iter-nonnull.rs b/tests/codegen/slice-iter-nonnull.rs index 98a1b961a6443..87907e7ad0a30 100644 --- a/tests/codegen/slice-iter-nonnull.rs +++ b/tests/codegen/slice-iter-nonnull.rs @@ -14,11 +14,11 @@ // CHECK-LABEL: @slice_iter_next( #[no_mangle] pub fn slice_iter_next<'a>(it: &mut std::slice::Iter<'a, u32>) -> Option<&'a u32> { - // CHECK: %[[ENDP:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %it, {{i32 4|i64 8}} - // CHECK: %[[END:.+]] = load ptr, ptr %[[ENDP]] + // CHECK: %[[START:.+]] = load ptr, ptr %it, // CHECK-SAME: !nonnull // CHECK-SAME: !noundef - // CHECK: %[[START:.+]] = load ptr, ptr %it, + // CHECK: %[[ENDP:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %it, {{i32 4|i64 8}} + // CHECK: %[[END:.+]] = load ptr, ptr %[[ENDP]] // CHECK-SAME: !nonnull // CHECK-SAME: !noundef // CHECK: icmp eq ptr %[[START]], %[[END]] diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index b7a9b4a1fe01c..e12f97073d0fd 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -4,28 +4,30 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::slice::Iter<'_, T>; - let mut _12: std::iter::Enumerate>; - let mut _13: std::iter::Enumerate>; - let mut _21: std::option::Option<(usize, &T)>; - let mut _24: &impl Fn(usize, &T); - let mut _25: (usize, &T); - let _26: (); + let mut _11: std::ptr::NonNull; + let mut _12: *const T; + let mut _13: usize; + let mut _33: std::option::Option<(usize, &T)>; + let mut _36: &impl Fn(usize, &T); + let mut _37: (usize, &T); + let _38: (); scope 1 { - debug iter => _13; - let _22: usize; - let _23: &T; + debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; + debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _12; + debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; + debug ((iter: Enumerate>).1: usize) => _13; + let _34: usize; + let _35: &T; scope 2 { - debug i => _22; - debug x => _23; + debug i => _34; + debug x => _35; } scope 19 (inlined > as Iterator>::next) { - let mut _14: &mut std::slice::Iter<'_, T>; - let mut _15: std::option::Option<&T>; - let mut _19: (usize, bool); - let mut _20: (usize, &T); + let mut _27: std::option::Option<&T>; + let mut _31: (usize, bool); + let mut _32: (usize, &T); scope 20 { - let _18: usize; + let _30: usize; scope 25 { } } @@ -40,11 +42,58 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 26 (inlined as Try>::branch) { - let mut _16: isize; - let _17: &T; + let mut _28: isize; + let _29: &T; scope 27 { } } + scope 29 (inlined as Iterator>::next) { + let _14: std::ptr::NonNull; + let _16: std::ptr::NonNull; + let mut _19: bool; + let mut _22: std::ptr::NonNull; + let mut _24: bool; + let mut _25: *const u8; + let mut _26: *const T; + scope 30 { + let _15: *const T; + scope 31 { + let _23: *const u8; + scope 32 { + scope 33 (inlined null::) { + scope 34 (inlined without_provenance::<()>) { + scope 35 (inlined without_provenance_mut::<()>) { + } + } + scope 36 (inlined std::ptr::from_raw_parts::) { + } + } + scope 37 (inlined std::ptr::const_ptr::::wrapping_sub) { + scope 38 (inlined core::num::::wrapping_neg) { + scope 39 (inlined core::num::::wrapping_sub) { + } + } + scope 40 (inlined std::ptr::const_ptr::::wrapping_offset) { + } + } + } + scope 41 (inlined as PartialEq>::eq) { + let mut _17: *mut T; + let mut _18: *mut T; + scope 42 (inlined NonNull::::as_ptr) { + } + scope 43 (inlined NonNull::::as_ptr) { + } + } + scope 44 (inlined NonNull::::add) { + let mut _20: *const T; + let mut _21: *const T; + scope 45 (inlined NonNull::::as_ptr) { + } + } + } + } + } } } scope 3 (inlined core::slice::::iter) { @@ -89,9 +138,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb0: { - StorageLive(_11); StorageLive(_3); - StorageLive(_6); StorageLive(_4); _3 = PtrMetadata(copy _1); _4 = &raw const (*_1); @@ -120,86 +167,162 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb3: { - StorageLive(_10); _10 = copy _9; - _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> }; - StorageDead(_10); StorageDead(_9); StorageDead(_4); - StorageDead(_6); StorageDead(_3); - _12 = Enumerate::> { iter: copy _11, count: const 0_usize }; - StorageDead(_11); + StorageLive(_11); + StorageLive(_12); StorageLive(_13); - _13 = copy _12; + _11 = copy _6; + _12 = copy _10; + _13 = const 0_usize; goto -> bb4; } bb4: { - StorageLive(_21); - StorageLive(_18); - StorageLive(_19); - StorageLive(_15); + StorageLive(_33); + StorageLive(_30); + StorageLive(_31); + StorageLive(_27); StorageLive(_14); - _14 = &mut (_13.0: std::slice::Iter<'_, T>); - _15 = as Iterator>::next(move _14) -> [return: bb5, unwind unreachable]; + StorageLive(_15); + StorageLive(_23); + StorageLive(_16); + _14 = copy _11; + _15 = copy _12; + switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - StorageDead(_14); - StorageLive(_16); - _16 = discriminant(_15); - switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb11]; + StorageLive(_19); + _16 = copy _15 as std::ptr::NonNull (Transmute); + StorageLive(_17); + _17 = copy _14 as *mut T (Transmute); + StorageLive(_18); + _18 = copy _16 as *mut T (Transmute); + _19 = Eq(move _17, move _18); + StorageDead(_18); + StorageDead(_17); + switchInt(move _19) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_16); - StorageDead(_15); StorageDead(_19); - StorageDead(_18); + StorageLive(_22); + StorageLive(_21); + StorageLive(_20); + _20 = copy _14 as *const T (Transmute); + _21 = Offset(move _20, const 1_usize); + StorageDead(_20); + _22 = NonNull:: { pointer: move _21 }; StorageDead(_21); - StorageDead(_13); - drop(_2) -> [return: bb7, unwind unreachable]; + _11 = move _22; + StorageDead(_22); + goto -> bb11; } bb7: { - return; + StorageDead(_19); + StorageDead(_16); + StorageDead(_23); + StorageDead(_15); + StorageDead(_14); + StorageLive(_28); + goto -> bb17; } bb8: { - _17 = move ((_15 as Some).0: &T); - StorageDead(_16); - StorageDead(_15); - _18 = copy (_13.1: usize); - _19 = AddWithOverflow(copy (_13.1: usize), const 1_usize); - assert(!move (_19.1: bool), "attempt to compute `{} + {}`, which would overflow", copy (_13.1: usize), const 1_usize) -> [success: bb9, unwind unreachable]; + _23 = copy _15 as *const u8 (PtrToPtr); + StorageLive(_24); + _24 = Eq(copy _23, const {0x0 as *const u8}); + switchInt(move _24) -> [0: bb9, otherwise: bb16]; } bb9: { - (_13.1: usize) = move (_19.0: usize); - StorageLive(_20); - _20 = (copy _18, copy _17); - _21 = Option::<(usize, &T)>::Some(move _20); - StorageDead(_20); - StorageDead(_19); - StorageDead(_18); - _22 = copy (((_21 as Some).0: (usize, &T)).0: usize); - _23 = copy (((_21 as Some).0: (usize, &T)).1: &T); - StorageLive(_24); - _24 = &_2; + StorageDead(_24); StorageLive(_25); - _25 = (copy _22, copy _23); - _26 = >::call(move _24, move _25) -> [return: bb10, unwind unreachable]; + _25 = arith_offset::(move _23, const -1_isize) -> [return: bb10, unwind unreachable]; } bb10: { + _26 = move _25 as *const T (PtrToPtr); StorageDead(_25); - StorageDead(_24); - StorageDead(_21); - goto -> bb4; + _12 = copy _26; + goto -> bb11; } bb11: { + _27 = copy _14 as std::option::Option<&T> (Transmute); + StorageDead(_16); + StorageDead(_23); + StorageDead(_15); + StorageDead(_14); + StorageLive(_28); + _28 = discriminant(_27); + switchInt(move _28) -> [0: bb17, 1: bb12, otherwise: bb15]; + } + + bb12: { + _29 = move ((_27 as Some).0: &T); + StorageDead(_28); + StorageDead(_27); + _30 = copy _13; + _31 = AddWithOverflow(copy _13, const 1_usize); + assert(!move (_31.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb13, unwind unreachable]; + } + + bb13: { + _13 = move (_31.0: usize); + StorageLive(_32); + _32 = (copy _30, copy _29); + _33 = Option::<(usize, &T)>::Some(move _32); + StorageDead(_32); + StorageDead(_31); + StorageDead(_30); + _34 = copy (((_33 as Some).0: (usize, &T)).0: usize); + _35 = copy (((_33 as Some).0: (usize, &T)).1: &T); + StorageLive(_36); + _36 = &_2; + StorageLive(_37); + _37 = (copy _34, copy _35); + _38 = >::call(move _36, move _37) -> [return: bb14, unwind unreachable]; + } + + bb14: { + StorageDead(_37); + StorageDead(_36); + StorageDead(_33); + goto -> bb4; + } + + bb15: { unreachable; } + + bb16: { + StorageDead(_24); + StorageDead(_16); + StorageDead(_23); + StorageDead(_15); + StorageDead(_14); + StorageLive(_28); + goto -> bb17; + } + + bb17: { + StorageDead(_28); + StorageDead(_27); + StorageDead(_31); + StorageDead(_30); + StorageDead(_33); + StorageDead(_11); + StorageDead(_12); + StorageDead(_13); + drop(_2) -> [return: bb18, unwind unreachable]; + } + + bb18: { + return; + } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir index dc13bb23c310a..88d0aa7f6240d 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -4,19 +4,67 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::slice::Iter<'_, T>; - let mut _12: std::slice::Iter<'_, T>; - let mut _13: &mut std::slice::Iter<'_, T>; - let mut _14: std::option::Option<&T>; - let mut _15: isize; - let mut _17: &impl Fn(&T); - let mut _18: (&T,); - let _19: (); + let mut _11: std::ptr::NonNull; + let mut _12: *const T; + let mut _26: std::option::Option<&T>; + let mut _27: isize; + let mut _29: &impl Fn(&T); + let mut _30: (&T,); + let _31: (); scope 1 { - debug iter => _12; - let _16: &T; + debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; + debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12; + debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; + let _28: &T; scope 2 { - debug x => _16; + debug x => _28; + } + scope 17 (inlined as Iterator>::next) { + let _13: std::ptr::NonNull; + let _15: std::ptr::NonNull; + let mut _18: bool; + let mut _21: std::ptr::NonNull; + let mut _23: bool; + let mut _24: *const u8; + let mut _25: *const T; + scope 18 { + let _14: *const T; + scope 19 { + let _22: *const u8; + scope 20 { + scope 21 (inlined null::) { + scope 22 (inlined without_provenance::<()>) { + scope 23 (inlined without_provenance_mut::<()>) { + } + } + scope 24 (inlined std::ptr::from_raw_parts::) { + } + } + scope 25 (inlined std::ptr::const_ptr::::wrapping_sub) { + scope 26 (inlined core::num::::wrapping_neg) { + scope 27 (inlined core::num::::wrapping_sub) { + } + } + scope 28 (inlined std::ptr::const_ptr::::wrapping_offset) { + } + } + } + scope 29 (inlined as PartialEq>::eq) { + let mut _16: *mut T; + let mut _17: *mut T; + scope 30 (inlined NonNull::::as_ptr) { + } + scope 31 (inlined NonNull::::as_ptr) { + } + } + scope 32 (inlined NonNull::::add) { + let mut _19: *const T; + let mut _20: *const T; + scope 33 (inlined NonNull::::as_ptr) { + } + } + } + } } } scope 3 (inlined core::slice::::iter) { @@ -58,7 +106,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb0: { StorageLive(_3); - StorageLive(_6); StorageLive(_4); _3 = PtrMetadata(copy _1); _4 = &raw const (*_1); @@ -87,57 +134,132 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb3: { - StorageLive(_10); _10 = copy _9; - _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> }; - StorageDead(_10); StorageDead(_9); StorageDead(_4); - StorageDead(_6); StorageDead(_3); + StorageLive(_11); StorageLive(_12); - _12 = copy _11; + _11 = copy _6; + _12 = copy _10; goto -> bb4; } bb4: { + StorageLive(_26); + StorageLive(_13); StorageLive(_14); - _13 = &mut _12; - _14 = as Iterator>::next(move _13) -> [return: bb5, unwind unreachable]; + StorageLive(_22); + StorageLive(_15); + _13 = copy _11; + _14 = copy _12; + switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - _15 = discriminant(_14); - switchInt(move _15) -> [0: bb6, 1: bb8, otherwise: bb10]; + StorageLive(_18); + _15 = copy _14 as std::ptr::NonNull (Transmute); + StorageLive(_16); + _16 = copy _13 as *mut T (Transmute); + StorageLive(_17); + _17 = copy _15 as *mut T (Transmute); + _18 = Eq(move _16, move _17); + StorageDead(_17); + StorageDead(_16); + switchInt(move _18) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_14); - StorageDead(_12); - drop(_2) -> [return: bb7, unwind unreachable]; + StorageDead(_18); + StorageLive(_21); + StorageLive(_20); + StorageLive(_19); + _19 = copy _13 as *const T (Transmute); + _20 = Offset(move _19, const 1_usize); + StorageDead(_19); + _21 = NonNull:: { pointer: move _20 }; + StorageDead(_20); + _11 = move _21; + StorageDead(_21); + goto -> bb11; } bb7: { - return; + StorageDead(_18); + StorageDead(_15); + StorageDead(_22); + StorageDead(_14); + StorageDead(_13); + goto -> bb16; } bb8: { - _16 = copy ((_14 as Some).0: &T); - StorageLive(_17); - _17 = &_2; - StorageLive(_18); - _18 = (copy _16,); - _19 = >::call(move _17, move _18) -> [return: bb9, unwind unreachable]; + _22 = copy _14 as *const u8 (PtrToPtr); + StorageLive(_23); + _23 = Eq(copy _22, const {0x0 as *const u8}); + switchInt(move _23) -> [0: bb9, otherwise: bb15]; } bb9: { - StorageDead(_18); - StorageDead(_17); + StorageDead(_23); + StorageLive(_24); + _24 = arith_offset::(move _22, const -1_isize) -> [return: bb10, unwind unreachable]; + } + + bb10: { + _25 = move _24 as *const T (PtrToPtr); + StorageDead(_24); + _12 = copy _25; + goto -> bb11; + } + + bb11: { + _26 = copy _13 as std::option::Option<&T> (Transmute); + StorageDead(_15); + StorageDead(_22); StorageDead(_14); + StorageDead(_13); + _27 = discriminant(_26); + switchInt(move _27) -> [0: bb16, 1: bb12, otherwise: bb14]; + } + + bb12: { + _28 = copy ((_26 as Some).0: &T); + StorageLive(_29); + _29 = &_2; + StorageLive(_30); + _30 = (copy _28,); + _31 = >::call(move _29, move _30) -> [return: bb13, unwind unreachable]; + } + + bb13: { + StorageDead(_30); + StorageDead(_29); + StorageDead(_26); goto -> bb4; } - bb10: { + bb14: { unreachable; } + + bb15: { + StorageDead(_23); + StorageDead(_15); + StorageDead(_22); + StorageDead(_14); + StorageDead(_13); + goto -> bb16; + } + + bb16: { + StorageDead(_26); + StorageDead(_11); + StorageDead(_12); + drop(_2) -> [return: bb17, unwind unreachable]; + } + + bb17: { + return; + } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir index 3f1e0e0f746b4..f34edc4775d38 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -4,19 +4,67 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::slice::Iter<'_, T>; - let mut _12: std::slice::Iter<'_, T>; - let mut _13: &mut std::slice::Iter<'_, T>; - let mut _14: std::option::Option<&T>; - let mut _15: isize; - let mut _17: &impl Fn(&T); - let mut _18: (&T,); - let _19: (); + let mut _11: std::ptr::NonNull; + let mut _12: *const T; + let mut _26: std::option::Option<&T>; + let mut _27: isize; + let mut _29: &impl Fn(&T); + let mut _30: (&T,); + let _31: (); scope 1 { - debug iter => _12; - let _16: &T; + debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; + debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12; + debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; + let _28: &T; scope 2 { - debug x => _16; + debug x => _28; + } + scope 17 (inlined as Iterator>::next) { + let _13: std::ptr::NonNull; + let _15: std::ptr::NonNull; + let mut _18: bool; + let mut _21: std::ptr::NonNull; + let mut _23: bool; + let mut _24: *const u8; + let mut _25: *const T; + scope 18 { + let _14: *const T; + scope 19 { + let _22: *const u8; + scope 20 { + scope 21 (inlined null::) { + scope 22 (inlined without_provenance::<()>) { + scope 23 (inlined without_provenance_mut::<()>) { + } + } + scope 24 (inlined std::ptr::from_raw_parts::) { + } + } + scope 25 (inlined std::ptr::const_ptr::::wrapping_sub) { + scope 26 (inlined core::num::::wrapping_neg) { + scope 27 (inlined core::num::::wrapping_sub) { + } + } + scope 28 (inlined std::ptr::const_ptr::::wrapping_offset) { + } + } + } + scope 29 (inlined as PartialEq>::eq) { + let mut _16: *mut T; + let mut _17: *mut T; + scope 30 (inlined NonNull::::as_ptr) { + } + scope 31 (inlined NonNull::::as_ptr) { + } + } + scope 32 (inlined NonNull::::add) { + let mut _19: *const T; + let mut _20: *const T; + scope 33 (inlined NonNull::::as_ptr) { + } + } + } + } } } scope 3 (inlined core::slice::::iter) { @@ -58,7 +106,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb0: { StorageLive(_3); - StorageLive(_6); StorageLive(_4); _3 = PtrMetadata(copy _1); _4 = &raw const (*_1); @@ -87,65 +134,140 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb3: { - StorageLive(_10); _10 = copy _9; - _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> }; - StorageDead(_10); StorageDead(_9); StorageDead(_4); - StorageDead(_6); StorageDead(_3); + StorageLive(_11); StorageLive(_12); - _12 = copy _11; + _11 = copy _6; + _12 = copy _10; goto -> bb4; } bb4: { + StorageLive(_26); + StorageLive(_13); StorageLive(_14); - _13 = &mut _12; - _14 = as Iterator>::next(move _13) -> [return: bb5, unwind: bb11]; + StorageLive(_22); + StorageLive(_15); + _13 = copy _11; + _14 = copy _12; + switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - _15 = discriminant(_14); - switchInt(move _15) -> [0: bb6, 1: bb8, otherwise: bb10]; + StorageLive(_18); + _15 = copy _14 as std::ptr::NonNull (Transmute); + StorageLive(_16); + _16 = copy _13 as *mut T (Transmute); + StorageLive(_17); + _17 = copy _15 as *mut T (Transmute); + _18 = Eq(move _16, move _17); + StorageDead(_17); + StorageDead(_16); + switchInt(move _18) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_14); - StorageDead(_12); - drop(_2) -> [return: bb7, unwind continue]; + StorageDead(_18); + StorageLive(_21); + StorageLive(_20); + StorageLive(_19); + _19 = copy _13 as *const T (Transmute); + _20 = Offset(move _19, const 1_usize); + StorageDead(_19); + _21 = NonNull:: { pointer: move _20 }; + StorageDead(_20); + _11 = move _21; + StorageDead(_21); + goto -> bb11; } bb7: { - return; + StorageDead(_18); + StorageDead(_15); + StorageDead(_22); + StorageDead(_14); + StorageDead(_13); + goto -> bb18; } bb8: { - _16 = copy ((_14 as Some).0: &T); - StorageLive(_17); - _17 = &_2; - StorageLive(_18); - _18 = (copy _16,); - _19 = >::call(move _17, move _18) -> [return: bb9, unwind: bb11]; + _22 = copy _14 as *const u8 (PtrToPtr); + StorageLive(_23); + _23 = Eq(copy _22, const {0x0 as *const u8}); + switchInt(move _23) -> [0: bb9, otherwise: bb17]; } bb9: { - StorageDead(_18); - StorageDead(_17); + StorageDead(_23); + StorageLive(_24); + _24 = arith_offset::(move _22, const -1_isize) -> [return: bb10, unwind unreachable]; + } + + bb10: { + _25 = move _24 as *const T (PtrToPtr); + StorageDead(_24); + _12 = copy _25; + goto -> bb11; + } + + bb11: { + _26 = copy _13 as std::option::Option<&T> (Transmute); + StorageDead(_15); + StorageDead(_22); StorageDead(_14); + StorageDead(_13); + _27 = discriminant(_26); + switchInt(move _27) -> [0: bb18, 1: bb12, otherwise: bb16]; + } + + bb12: { + _28 = copy ((_26 as Some).0: &T); + StorageLive(_29); + _29 = &_2; + StorageLive(_30); + _30 = (copy _28,); + _31 = >::call(move _29, move _30) -> [return: bb13, unwind: bb14]; + } + + bb13: { + StorageDead(_30); + StorageDead(_29); + StorageDead(_26); goto -> bb4; } - bb10: { + bb14 (cleanup): { + drop(_2) -> [return: bb15, unwind terminate(cleanup)]; + } + + bb15 (cleanup): { + resume; + } + + bb16: { unreachable; } - bb11 (cleanup): { - drop(_2) -> [return: bb12, unwind terminate(cleanup)]; + bb17: { + StorageDead(_23); + StorageDead(_15); + StorageDead(_22); + StorageDead(_14); + StorageDead(_13); + goto -> bb18; } - bb12 (cleanup): { - resume; + bb18: { + StorageDead(_26); + StorageDead(_11); + StorageDead(_12); + drop(_2) -> [return: bb19, unwind continue]; + } + + bb19: { + return; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir index 8edac638ccdd6..975a8a11fb212 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir @@ -3,12 +3,134 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { debug it => _1; let mut _0: std::option::Option<&T>; + scope 1 (inlined as Iterator>::next) { + let _2: std::ptr::NonNull; + let _4: std::ptr::NonNull; + let mut _7: bool; + let mut _10: std::ptr::NonNull; + let mut _12: bool; + let mut _13: *const u8; + let mut _14: *const T; + scope 2 { + let _3: *const T; + scope 3 { + let _11: *const u8; + scope 4 { + scope 5 (inlined null::) { + scope 6 (inlined without_provenance::<()>) { + scope 7 (inlined without_provenance_mut::<()>) { + } + } + scope 8 (inlined std::ptr::from_raw_parts::) { + } + } + scope 9 (inlined std::ptr::const_ptr::::wrapping_sub) { + scope 10 (inlined core::num::::wrapping_neg) { + scope 11 (inlined core::num::::wrapping_sub) { + } + } + scope 12 (inlined std::ptr::const_ptr::::wrapping_offset) { + } + } + } + scope 13 (inlined as PartialEq>::eq) { + let mut _5: *mut T; + let mut _6: *mut T; + scope 14 (inlined NonNull::::as_ptr) { + } + scope 15 (inlined NonNull::::as_ptr) { + } + } + scope 16 (inlined NonNull::::add) { + let mut _8: *const T; + let mut _9: *const T; + scope 17 (inlined NonNull::::as_ptr) { + } + } + } + } + } bb0: { - _0 = as Iterator>::next(move _1) -> [return: bb1, unwind unreachable]; + StorageLive(_2); + StorageLive(_3); + StorageLive(_11); + StorageLive(_4); + _2 = copy ((*_1).0: std::ptr::NonNull); + _3 = copy ((*_1).1: *const T); + switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb4]; } bb1: { + StorageLive(_7); + _4 = copy _3 as std::ptr::NonNull (Transmute); + StorageLive(_5); + _5 = copy _2 as *mut T (Transmute); + StorageLive(_6); + _6 = copy _4 as *mut T (Transmute); + _7 = Eq(move _5, move _6); + StorageDead(_6); + StorageDead(_5); + switchInt(move _7) -> [0: bb2, otherwise: bb3]; + } + + bb2: { + StorageDead(_7); + StorageLive(_10); + StorageLive(_9); + StorageLive(_8); + _8 = copy _2 as *const T (Transmute); + _9 = Offset(move _8, const 1_usize); + StorageDead(_8); + _10 = NonNull:: { pointer: move _9 }; + StorageDead(_9); + ((*_1).0: std::ptr::NonNull) = move _10; + StorageDead(_10); + goto -> bb7; + } + + bb3: { + _0 = const {transmute(0x0000000000000000): Option<&T>}; + StorageDead(_7); + goto -> bb9; + } + + bb4: { + _11 = copy _3 as *const u8 (PtrToPtr); + StorageLive(_12); + _12 = Eq(copy _11, const {0x0 as *const u8}); + switchInt(move _12) -> [0: bb5, otherwise: bb8]; + } + + bb5: { + StorageDead(_12); + StorageLive(_13); + _13 = arith_offset::(move _11, const -1_isize) -> [return: bb6, unwind unreachable]; + } + + bb6: { + _14 = move _13 as *const T (PtrToPtr); + StorageDead(_13); + ((*_1).1: *const T) = copy _14; + goto -> bb7; + } + + bb7: { + _0 = copy _2 as std::option::Option<&T> (Transmute); + goto -> bb9; + } + + bb8: { + _0 = const {transmute(0x0000000000000000): Option<&T>}; + StorageDead(_12); + goto -> bb9; + } + + bb9: { + StorageDead(_4); + StorageDead(_11); + StorageDead(_3); + StorageDead(_2); return; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir index fdde07173437b..975a8a11fb212 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir @@ -3,12 +3,134 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { debug it => _1; let mut _0: std::option::Option<&T>; + scope 1 (inlined as Iterator>::next) { + let _2: std::ptr::NonNull; + let _4: std::ptr::NonNull; + let mut _7: bool; + let mut _10: std::ptr::NonNull; + let mut _12: bool; + let mut _13: *const u8; + let mut _14: *const T; + scope 2 { + let _3: *const T; + scope 3 { + let _11: *const u8; + scope 4 { + scope 5 (inlined null::) { + scope 6 (inlined without_provenance::<()>) { + scope 7 (inlined without_provenance_mut::<()>) { + } + } + scope 8 (inlined std::ptr::from_raw_parts::) { + } + } + scope 9 (inlined std::ptr::const_ptr::::wrapping_sub) { + scope 10 (inlined core::num::::wrapping_neg) { + scope 11 (inlined core::num::::wrapping_sub) { + } + } + scope 12 (inlined std::ptr::const_ptr::::wrapping_offset) { + } + } + } + scope 13 (inlined as PartialEq>::eq) { + let mut _5: *mut T; + let mut _6: *mut T; + scope 14 (inlined NonNull::::as_ptr) { + } + scope 15 (inlined NonNull::::as_ptr) { + } + } + scope 16 (inlined NonNull::::add) { + let mut _8: *const T; + let mut _9: *const T; + scope 17 (inlined NonNull::::as_ptr) { + } + } + } + } + } bb0: { - _0 = as Iterator>::next(move _1) -> [return: bb1, unwind continue]; + StorageLive(_2); + StorageLive(_3); + StorageLive(_11); + StorageLive(_4); + _2 = copy ((*_1).0: std::ptr::NonNull); + _3 = copy ((*_1).1: *const T); + switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb4]; } bb1: { + StorageLive(_7); + _4 = copy _3 as std::ptr::NonNull (Transmute); + StorageLive(_5); + _5 = copy _2 as *mut T (Transmute); + StorageLive(_6); + _6 = copy _4 as *mut T (Transmute); + _7 = Eq(move _5, move _6); + StorageDead(_6); + StorageDead(_5); + switchInt(move _7) -> [0: bb2, otherwise: bb3]; + } + + bb2: { + StorageDead(_7); + StorageLive(_10); + StorageLive(_9); + StorageLive(_8); + _8 = copy _2 as *const T (Transmute); + _9 = Offset(move _8, const 1_usize); + StorageDead(_8); + _10 = NonNull:: { pointer: move _9 }; + StorageDead(_9); + ((*_1).0: std::ptr::NonNull) = move _10; + StorageDead(_10); + goto -> bb7; + } + + bb3: { + _0 = const {transmute(0x0000000000000000): Option<&T>}; + StorageDead(_7); + goto -> bb9; + } + + bb4: { + _11 = copy _3 as *const u8 (PtrToPtr); + StorageLive(_12); + _12 = Eq(copy _11, const {0x0 as *const u8}); + switchInt(move _12) -> [0: bb5, otherwise: bb8]; + } + + bb5: { + StorageDead(_12); + StorageLive(_13); + _13 = arith_offset::(move _11, const -1_isize) -> [return: bb6, unwind unreachable]; + } + + bb6: { + _14 = move _13 as *const T (PtrToPtr); + StorageDead(_13); + ((*_1).1: *const T) = copy _14; + goto -> bb7; + } + + bb7: { + _0 = copy _2 as std::option::Option<&T> (Transmute); + goto -> bb9; + } + + bb8: { + _0 = const {transmute(0x0000000000000000): Option<&T>}; + StorageDead(_12); + goto -> bb9; + } + + bb9: { + StorageDead(_4); + StorageDead(_11); + StorageDead(_3); + StorageDead(_2); return; } } From 3a62c70051e40fbcfc7e5b9ad60cb9cfc4dddcee Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Thu, 13 Feb 2025 22:28:29 -0800 Subject: [PATCH 2/4] Save another BB by using `SubUnchecked` instead of a call to `arith_offset` Probably reasonable anyway since it more obviously drops provenance. --- library/core/src/slice/iter.rs | 2 +- library/core/src/slice/iter/macros.rs | 8 +- ...ated_loop.PreCodegen.after.panic-abort.mir | 182 ++++++++---------- ...ward_loop.PreCodegen.after.panic-abort.mir | 122 ++++++------ ...ard_loop.PreCodegen.after.panic-unwind.mir | 134 ++++++------- ...iter_next.PreCodegen.after.panic-abort.mir | 64 +++--- ...ter_next.PreCodegen.after.panic-unwind.mir | 64 +++--- 7 files changed, 256 insertions(+), 320 deletions(-) diff --git a/library/core/src/slice/iter.rs b/library/core/src/slice/iter.rs index 5a87f1327dd74..a687ed7129dc8 100644 --- a/library/core/src/slice/iter.rs +++ b/library/core/src/slice/iter.rs @@ -11,7 +11,7 @@ use crate::iter::{ use crate::marker::PhantomData; use crate::mem::{self, SizedTypeProperties}; use crate::num::NonZero; -use crate::ptr::{NonNull, null, without_provenance, without_provenance_mut}; +use crate::ptr::{NonNull, without_provenance, without_provenance_mut}; use crate::{cmp, fmt}; #[stable(feature = "boxed_slice_into_iter", since = "1.80.0")] diff --git a/library/core/src/slice/iter/macros.rs b/library/core/src/slice/iter/macros.rs index 74dfb8634f2ff..101f3157651f6 100644 --- a/library/core/src/slice/iter/macros.rs +++ b/library/core/src/slice/iter/macros.rs @@ -162,13 +162,13 @@ macro_rules! iterator { // SAFETY: Type invariants. unsafe { if T::IS_ZST { - let byte_end = end_or_len as *const u8; - if byte_end == null() { + let len = end_or_len.addr(); + if len == 0 { return None; } - self.end_or_len = byte_end.wrapping_sub(1) as _; + self.end_or_len = without_provenance_mut(len.unchecked_sub(1)); } else { - if ptr == crate::intrinsics::transmute::<*const T, NonNull>(end_or_len) { + if ptr == crate::intrinsics::transmute::<$ptr, NonNull>(end_or_len) { return None; } self.ptr = ptr.add(1); diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index e12f97073d0fd..80b96e01041e6 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -7,27 +7,27 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let mut _11: std::ptr::NonNull; let mut _12: *const T; let mut _13: usize; - let mut _33: std::option::Option<(usize, &T)>; - let mut _36: &impl Fn(usize, &T); - let mut _37: (usize, &T); - let _38: (); + let mut _31: std::option::Option<(usize, &T)>; + let mut _34: &impl Fn(usize, &T); + let mut _35: (usize, &T); + let _36: (); scope 1 { debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _12; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; debug ((iter: Enumerate>).1: usize) => _13; - let _34: usize; - let _35: &T; + let _32: usize; + let _33: &T; scope 2 { - debug i => _34; - debug x => _35; + debug i => _32; + debug x => _33; } scope 19 (inlined > as Iterator>::next) { - let mut _27: std::option::Option<&T>; - let mut _31: (usize, bool); - let mut _32: (usize, &T); + let mut _25: std::option::Option<&T>; + let mut _29: (usize, bool); + let mut _30: (usize, &T); scope 20 { - let _30: usize; + let _28: usize; scope 25 { } } @@ -42,8 +42,8 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 26 (inlined as Try>::branch) { - let mut _28: isize; - let _29: &T; + let mut _26: isize; + let _27: &T; scope 27 { } } @@ -52,43 +52,37 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let _16: std::ptr::NonNull; let mut _19: bool; let mut _22: std::ptr::NonNull; - let mut _24: bool; - let mut _25: *const u8; - let mut _26: *const T; + let mut _24: usize; scope 30 { let _15: *const T; scope 31 { - let _23: *const u8; + let _23: usize; scope 32 { - scope 33 (inlined null::) { - scope 34 (inlined without_provenance::<()>) { - scope 35 (inlined without_provenance_mut::<()>) { + scope 35 (inlined core::num::::unchecked_sub) { + scope 36 (inlined core::ub_checks::check_language_ub) { + scope 37 (inlined core::ub_checks::check_language_ub::runtime) { } } - scope 36 (inlined std::ptr::from_raw_parts::) { - } } - scope 37 (inlined std::ptr::const_ptr::::wrapping_sub) { - scope 38 (inlined core::num::::wrapping_neg) { - scope 39 (inlined core::num::::wrapping_sub) { - } - } - scope 40 (inlined std::ptr::const_ptr::::wrapping_offset) { - } + scope 38 (inlined without_provenance_mut::) { } } - scope 41 (inlined as PartialEq>::eq) { + scope 33 (inlined std::ptr::const_ptr::::addr) { + scope 34 (inlined std::ptr::const_ptr::::cast::<()>) { + } + } + scope 39 (inlined as PartialEq>::eq) { let mut _17: *mut T; let mut _18: *mut T; - scope 42 (inlined NonNull::::as_ptr) { + scope 40 (inlined NonNull::::as_ptr) { } - scope 43 (inlined NonNull::::as_ptr) { + scope 41 (inlined NonNull::::as_ptr) { } } - scope 44 (inlined NonNull::::add) { + scope 42 (inlined NonNull::::add) { let mut _20: *const T; let mut _21: *const T; - scope 45 (inlined NonNull::::as_ptr) { + scope 43 (inlined NonNull::::as_ptr) { } } } @@ -181,13 +175,14 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb4: { - StorageLive(_33); - StorageLive(_30); StorageLive(_31); - StorageLive(_27); + StorageLive(_28); + StorageLive(_29); + StorageLive(_25); StorageLive(_14); StorageLive(_15); StorageLive(_23); + StorageLive(_24); StorageLive(_16); _14 = copy _11; _15 = copy _12; @@ -225,104 +220,97 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb7: { StorageDead(_19); StorageDead(_16); + StorageDead(_24); StorageDead(_23); StorageDead(_15); StorageDead(_14); - StorageLive(_28); - goto -> bb17; + StorageLive(_26); + goto -> bb12; } bb8: { - _23 = copy _15 as *const u8 (PtrToPtr); - StorageLive(_24); - _24 = Eq(copy _23, const {0x0 as *const u8}); - switchInt(move _24) -> [0: bb9, otherwise: bb16]; + _23 = copy _15 as usize (Transmute); + switchInt(copy _23) -> [0: bb9, otherwise: bb10]; } bb9: { + StorageDead(_16); StorageDead(_24); - StorageLive(_25); - _25 = arith_offset::(move _23, const -1_isize) -> [return: bb10, unwind unreachable]; + StorageDead(_23); + StorageDead(_15); + StorageDead(_14); + StorageLive(_26); + goto -> bb12; } bb10: { - _26 = move _25 as *const T (PtrToPtr); - StorageDead(_25); - _12 = copy _26; + _24 = SubUnchecked(copy _23, const 1_usize); + _12 = copy _24 as *const T (Transmute); goto -> bb11; } bb11: { - _27 = copy _14 as std::option::Option<&T> (Transmute); + _25 = copy _14 as std::option::Option<&T> (Transmute); StorageDead(_16); + StorageDead(_24); StorageDead(_23); StorageDead(_15); StorageDead(_14); - StorageLive(_28); - _28 = discriminant(_27); - switchInt(move _28) -> [0: bb17, 1: bb12, otherwise: bb15]; + StorageLive(_26); + _26 = discriminant(_25); + switchInt(move _26) -> [0: bb12, 1: bb14, otherwise: bb17]; } bb12: { - _29 = move ((_27 as Some).0: &T); + StorageDead(_26); + StorageDead(_25); + StorageDead(_29); StorageDead(_28); - StorageDead(_27); - _30 = copy _13; - _31 = AddWithOverflow(copy _13, const 1_usize); - assert(!move (_31.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb13, unwind unreachable]; + StorageDead(_31); + StorageDead(_11); + StorageDead(_12); + StorageDead(_13); + drop(_2) -> [return: bb13, unwind unreachable]; } bb13: { - _13 = move (_31.0: usize); - StorageLive(_32); - _32 = (copy _30, copy _29); - _33 = Option::<(usize, &T)>::Some(move _32); - StorageDead(_32); - StorageDead(_31); - StorageDead(_30); - _34 = copy (((_33 as Some).0: (usize, &T)).0: usize); - _35 = copy (((_33 as Some).0: (usize, &T)).1: &T); - StorageLive(_36); - _36 = &_2; - StorageLive(_37); - _37 = (copy _34, copy _35); - _38 = >::call(move _36, move _37) -> [return: bb14, unwind unreachable]; + return; } bb14: { - StorageDead(_37); - StorageDead(_36); - StorageDead(_33); - goto -> bb4; + _27 = move ((_25 as Some).0: &T); + StorageDead(_26); + StorageDead(_25); + _28 = copy _13; + _29 = AddWithOverflow(copy _13, const 1_usize); + assert(!move (_29.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb15, unwind unreachable]; } bb15: { - unreachable; + _13 = move (_29.0: usize); + StorageLive(_30); + _30 = (copy _28, copy _27); + _31 = Option::<(usize, &T)>::Some(move _30); + StorageDead(_30); + StorageDead(_29); + StorageDead(_28); + _32 = copy (((_31 as Some).0: (usize, &T)).0: usize); + _33 = copy (((_31 as Some).0: (usize, &T)).1: &T); + StorageLive(_34); + _34 = &_2; + StorageLive(_35); + _35 = (copy _32, copy _33); + _36 = >::call(move _34, move _35) -> [return: bb16, unwind unreachable]; } bb16: { - StorageDead(_24); - StorageDead(_16); - StorageDead(_23); - StorageDead(_15); - StorageDead(_14); - StorageLive(_28); - goto -> bb17; - } - - bb17: { - StorageDead(_28); - StorageDead(_27); + StorageDead(_35); + StorageDead(_34); StorageDead(_31); - StorageDead(_30); - StorageDead(_33); - StorageDead(_11); - StorageDead(_12); - StorageDead(_13); - drop(_2) -> [return: bb18, unwind unreachable]; + goto -> bb4; } - bb18: { - return; + bb17: { + unreachable; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir index 88d0aa7f6240d..c539170055d66 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -6,61 +6,55 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _0: (); let mut _11: std::ptr::NonNull; let mut _12: *const T; - let mut _26: std::option::Option<&T>; - let mut _27: isize; - let mut _29: &impl Fn(&T); - let mut _30: (&T,); - let _31: (); + let mut _24: std::option::Option<&T>; + let mut _25: isize; + let mut _27: &impl Fn(&T); + let mut _28: (&T,); + let _29: (); scope 1 { debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12; debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - let _28: &T; + let _26: &T; scope 2 { - debug x => _28; + debug x => _26; } scope 17 (inlined as Iterator>::next) { let _13: std::ptr::NonNull; let _15: std::ptr::NonNull; let mut _18: bool; let mut _21: std::ptr::NonNull; - let mut _23: bool; - let mut _24: *const u8; - let mut _25: *const T; + let mut _23: usize; scope 18 { let _14: *const T; scope 19 { - let _22: *const u8; + let _22: usize; scope 20 { - scope 21 (inlined null::) { - scope 22 (inlined without_provenance::<()>) { - scope 23 (inlined without_provenance_mut::<()>) { + scope 23 (inlined core::num::::unchecked_sub) { + scope 24 (inlined core::ub_checks::check_language_ub) { + scope 25 (inlined core::ub_checks::check_language_ub::runtime) { } } - scope 24 (inlined std::ptr::from_raw_parts::) { - } } - scope 25 (inlined std::ptr::const_ptr::::wrapping_sub) { - scope 26 (inlined core::num::::wrapping_neg) { - scope 27 (inlined core::num::::wrapping_sub) { - } - } - scope 28 (inlined std::ptr::const_ptr::::wrapping_offset) { - } + scope 26 (inlined without_provenance_mut::) { + } + } + scope 21 (inlined std::ptr::const_ptr::::addr) { + scope 22 (inlined std::ptr::const_ptr::::cast::<()>) { } } - scope 29 (inlined as PartialEq>::eq) { + scope 27 (inlined as PartialEq>::eq) { let mut _16: *mut T; let mut _17: *mut T; - scope 30 (inlined NonNull::::as_ptr) { + scope 28 (inlined NonNull::::as_ptr) { } - scope 31 (inlined NonNull::::as_ptr) { + scope 29 (inlined NonNull::::as_ptr) { } } - scope 32 (inlined NonNull::::add) { + scope 30 (inlined NonNull::::add) { let mut _19: *const T; let mut _20: *const T; - scope 33 (inlined NonNull::::as_ptr) { + scope 31 (inlined NonNull::::as_ptr) { } } } @@ -146,10 +140,11 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_26); + StorageLive(_24); StorageLive(_13); StorageLive(_14); StorageLive(_22); + StorageLive(_23); StorageLive(_15); _13 = copy _11; _14 = copy _12; @@ -187,79 +182,72 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb7: { StorageDead(_18); StorageDead(_15); + StorageDead(_23); StorageDead(_22); StorageDead(_14); StorageDead(_13); - goto -> bb16; + goto -> bb12; } bb8: { - _22 = copy _14 as *const u8 (PtrToPtr); - StorageLive(_23); - _23 = Eq(copy _22, const {0x0 as *const u8}); - switchInt(move _23) -> [0: bb9, otherwise: bb15]; + _22 = copy _14 as usize (Transmute); + switchInt(copy _22) -> [0: bb9, otherwise: bb10]; } bb9: { + StorageDead(_15); StorageDead(_23); - StorageLive(_24); - _24 = arith_offset::(move _22, const -1_isize) -> [return: bb10, unwind unreachable]; + StorageDead(_22); + StorageDead(_14); + StorageDead(_13); + goto -> bb12; } bb10: { - _25 = move _24 as *const T (PtrToPtr); - StorageDead(_24); - _12 = copy _25; + _23 = SubUnchecked(copy _22, const 1_usize); + _12 = copy _23 as *const T (Transmute); goto -> bb11; } bb11: { - _26 = copy _13 as std::option::Option<&T> (Transmute); + _24 = copy _13 as std::option::Option<&T> (Transmute); StorageDead(_15); + StorageDead(_23); StorageDead(_22); StorageDead(_14); StorageDead(_13); - _27 = discriminant(_26); - switchInt(move _27) -> [0: bb16, 1: bb12, otherwise: bb14]; + _25 = discriminant(_24); + switchInt(move _25) -> [0: bb12, 1: bb14, otherwise: bb16]; } bb12: { - _28 = copy ((_26 as Some).0: &T); - StorageLive(_29); - _29 = &_2; - StorageLive(_30); - _30 = (copy _28,); - _31 = >::call(move _29, move _30) -> [return: bb13, unwind unreachable]; + StorageDead(_24); + StorageDead(_11); + StorageDead(_12); + drop(_2) -> [return: bb13, unwind unreachable]; } bb13: { - StorageDead(_30); - StorageDead(_29); - StorageDead(_26); - goto -> bb4; + return; } bb14: { - unreachable; + _26 = copy ((_24 as Some).0: &T); + StorageLive(_27); + _27 = &_2; + StorageLive(_28); + _28 = (copy _26,); + _29 = >::call(move _27, move _28) -> [return: bb15, unwind unreachable]; } bb15: { - StorageDead(_23); - StorageDead(_15); - StorageDead(_22); - StorageDead(_14); - StorageDead(_13); - goto -> bb16; + StorageDead(_28); + StorageDead(_27); + StorageDead(_24); + goto -> bb4; } bb16: { - StorageDead(_26); - StorageDead(_11); - StorageDead(_12); - drop(_2) -> [return: bb17, unwind unreachable]; - } - - bb17: { - return; + unreachable; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir index f34edc4775d38..dc537ad87402f 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -6,61 +6,55 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _0: (); let mut _11: std::ptr::NonNull; let mut _12: *const T; - let mut _26: std::option::Option<&T>; - let mut _27: isize; - let mut _29: &impl Fn(&T); - let mut _30: (&T,); - let _31: (); + let mut _24: std::option::Option<&T>; + let mut _25: isize; + let mut _27: &impl Fn(&T); + let mut _28: (&T,); + let _29: (); scope 1 { debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12; debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - let _28: &T; + let _26: &T; scope 2 { - debug x => _28; + debug x => _26; } scope 17 (inlined as Iterator>::next) { let _13: std::ptr::NonNull; let _15: std::ptr::NonNull; let mut _18: bool; let mut _21: std::ptr::NonNull; - let mut _23: bool; - let mut _24: *const u8; - let mut _25: *const T; + let mut _23: usize; scope 18 { let _14: *const T; scope 19 { - let _22: *const u8; + let _22: usize; scope 20 { - scope 21 (inlined null::) { - scope 22 (inlined without_provenance::<()>) { - scope 23 (inlined without_provenance_mut::<()>) { + scope 23 (inlined core::num::::unchecked_sub) { + scope 24 (inlined core::ub_checks::check_language_ub) { + scope 25 (inlined core::ub_checks::check_language_ub::runtime) { } } - scope 24 (inlined std::ptr::from_raw_parts::) { - } } - scope 25 (inlined std::ptr::const_ptr::::wrapping_sub) { - scope 26 (inlined core::num::::wrapping_neg) { - scope 27 (inlined core::num::::wrapping_sub) { - } - } - scope 28 (inlined std::ptr::const_ptr::::wrapping_offset) { - } + scope 26 (inlined without_provenance_mut::) { + } + } + scope 21 (inlined std::ptr::const_ptr::::addr) { + scope 22 (inlined std::ptr::const_ptr::::cast::<()>) { } } - scope 29 (inlined as PartialEq>::eq) { + scope 27 (inlined as PartialEq>::eq) { let mut _16: *mut T; let mut _17: *mut T; - scope 30 (inlined NonNull::::as_ptr) { + scope 28 (inlined NonNull::::as_ptr) { } - scope 31 (inlined NonNull::::as_ptr) { + scope 29 (inlined NonNull::::as_ptr) { } } - scope 32 (inlined NonNull::::add) { + scope 30 (inlined NonNull::::add) { let mut _19: *const T; let mut _20: *const T; - scope 33 (inlined NonNull::::as_ptr) { + scope 31 (inlined NonNull::::as_ptr) { } } } @@ -146,10 +140,11 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_26); + StorageLive(_24); StorageLive(_13); StorageLive(_14); StorageLive(_22); + StorageLive(_23); StorageLive(_15); _13 = copy _11; _14 = copy _12; @@ -187,87 +182,80 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb7: { StorageDead(_18); StorageDead(_15); + StorageDead(_23); StorageDead(_22); StorageDead(_14); StorageDead(_13); - goto -> bb18; + goto -> bb12; } bb8: { - _22 = copy _14 as *const u8 (PtrToPtr); - StorageLive(_23); - _23 = Eq(copy _22, const {0x0 as *const u8}); - switchInt(move _23) -> [0: bb9, otherwise: bb17]; + _22 = copy _14 as usize (Transmute); + switchInt(copy _22) -> [0: bb9, otherwise: bb10]; } bb9: { + StorageDead(_15); StorageDead(_23); - StorageLive(_24); - _24 = arith_offset::(move _22, const -1_isize) -> [return: bb10, unwind unreachable]; + StorageDead(_22); + StorageDead(_14); + StorageDead(_13); + goto -> bb12; } bb10: { - _25 = move _24 as *const T (PtrToPtr); - StorageDead(_24); - _12 = copy _25; + _23 = SubUnchecked(copy _22, const 1_usize); + _12 = copy _23 as *const T (Transmute); goto -> bb11; } bb11: { - _26 = copy _13 as std::option::Option<&T> (Transmute); + _24 = copy _13 as std::option::Option<&T> (Transmute); StorageDead(_15); + StorageDead(_23); StorageDead(_22); StorageDead(_14); StorageDead(_13); - _27 = discriminant(_26); - switchInt(move _27) -> [0: bb18, 1: bb12, otherwise: bb16]; + _25 = discriminant(_24); + switchInt(move _25) -> [0: bb12, 1: bb14, otherwise: bb18]; } bb12: { - _28 = copy ((_26 as Some).0: &T); - StorageLive(_29); - _29 = &_2; - StorageLive(_30); - _30 = (copy _28,); - _31 = >::call(move _29, move _30) -> [return: bb13, unwind: bb14]; + StorageDead(_24); + StorageDead(_11); + StorageDead(_12); + drop(_2) -> [return: bb13, unwind continue]; } bb13: { - StorageDead(_30); - StorageDead(_29); - StorageDead(_26); - goto -> bb4; + return; } - bb14 (cleanup): { - drop(_2) -> [return: bb15, unwind terminate(cleanup)]; + bb14: { + _26 = copy ((_24 as Some).0: &T); + StorageLive(_27); + _27 = &_2; + StorageLive(_28); + _28 = (copy _26,); + _29 = >::call(move _27, move _28) -> [return: bb15, unwind: bb16]; } - bb15 (cleanup): { - resume; + bb15: { + StorageDead(_28); + StorageDead(_27); + StorageDead(_24); + goto -> bb4; } - bb16: { - unreachable; + bb16 (cleanup): { + drop(_2) -> [return: bb17, unwind terminate(cleanup)]; } - bb17: { - StorageDead(_23); - StorageDead(_15); - StorageDead(_22); - StorageDead(_14); - StorageDead(_13); - goto -> bb18; + bb17 (cleanup): { + resume; } bb18: { - StorageDead(_26); - StorageDead(_11); - StorageDead(_12); - drop(_2) -> [return: bb19, unwind continue]; - } - - bb19: { - return; + unreachable; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir index 975a8a11fb212..0fe469c21f742 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir @@ -8,43 +8,37 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { let _4: std::ptr::NonNull; let mut _7: bool; let mut _10: std::ptr::NonNull; - let mut _12: bool; - let mut _13: *const u8; - let mut _14: *const T; + let mut _12: usize; scope 2 { let _3: *const T; scope 3 { - let _11: *const u8; + let _11: usize; scope 4 { - scope 5 (inlined null::) { - scope 6 (inlined without_provenance::<()>) { - scope 7 (inlined without_provenance_mut::<()>) { + scope 7 (inlined core::num::::unchecked_sub) { + scope 8 (inlined core::ub_checks::check_language_ub) { + scope 9 (inlined core::ub_checks::check_language_ub::runtime) { } } - scope 8 (inlined std::ptr::from_raw_parts::) { - } } - scope 9 (inlined std::ptr::const_ptr::::wrapping_sub) { - scope 10 (inlined core::num::::wrapping_neg) { - scope 11 (inlined core::num::::wrapping_sub) { - } - } - scope 12 (inlined std::ptr::const_ptr::::wrapping_offset) { - } + scope 10 (inlined without_provenance_mut::) { } } - scope 13 (inlined as PartialEq>::eq) { + scope 5 (inlined std::ptr::const_ptr::::addr) { + scope 6 (inlined std::ptr::const_ptr::::cast::<()>) { + } + } + scope 11 (inlined as PartialEq>::eq) { let mut _5: *mut T; let mut _6: *mut T; - scope 14 (inlined NonNull::::as_ptr) { + scope 12 (inlined NonNull::::as_ptr) { } - scope 15 (inlined NonNull::::as_ptr) { + scope 13 (inlined NonNull::::as_ptr) { } } - scope 16 (inlined NonNull::::add) { + scope 14 (inlined NonNull::::add) { let mut _8: *const T; let mut _9: *const T; - scope 17 (inlined NonNull::::as_ptr) { + scope 15 (inlined NonNull::::as_ptr) { } } } @@ -55,6 +49,7 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { StorageLive(_2); StorageLive(_3); StorageLive(_11); + StorageLive(_12); StorageLive(_4); _2 = copy ((*_1).0: std::ptr::NonNull); _3 = copy ((*_1).1: *const T); @@ -92,42 +87,33 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { bb3: { _0 = const {transmute(0x0000000000000000): Option<&T>}; StorageDead(_7); - goto -> bb9; + goto -> bb8; } bb4: { - _11 = copy _3 as *const u8 (PtrToPtr); - StorageLive(_12); - _12 = Eq(copy _11, const {0x0 as *const u8}); - switchInt(move _12) -> [0: bb5, otherwise: bb8]; + _11 = copy _3 as usize (Transmute); + switchInt(copy _11) -> [0: bb5, otherwise: bb6]; } bb5: { - StorageDead(_12); - StorageLive(_13); - _13 = arith_offset::(move _11, const -1_isize) -> [return: bb6, unwind unreachable]; + _0 = const {transmute(0x0000000000000000): Option<&T>}; + goto -> bb8; } bb6: { - _14 = move _13 as *const T (PtrToPtr); - StorageDead(_13); - ((*_1).1: *const T) = copy _14; + _12 = SubUnchecked(copy _11, const 1_usize); + ((*_1).1: *const T) = copy _12 as *const T (Transmute); goto -> bb7; } bb7: { _0 = copy _2 as std::option::Option<&T> (Transmute); - goto -> bb9; + goto -> bb8; } bb8: { - _0 = const {transmute(0x0000000000000000): Option<&T>}; - StorageDead(_12); - goto -> bb9; - } - - bb9: { StorageDead(_4); + StorageDead(_12); StorageDead(_11); StorageDead(_3); StorageDead(_2); diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir index 975a8a11fb212..0fe469c21f742 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir @@ -8,43 +8,37 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { let _4: std::ptr::NonNull; let mut _7: bool; let mut _10: std::ptr::NonNull; - let mut _12: bool; - let mut _13: *const u8; - let mut _14: *const T; + let mut _12: usize; scope 2 { let _3: *const T; scope 3 { - let _11: *const u8; + let _11: usize; scope 4 { - scope 5 (inlined null::) { - scope 6 (inlined without_provenance::<()>) { - scope 7 (inlined without_provenance_mut::<()>) { + scope 7 (inlined core::num::::unchecked_sub) { + scope 8 (inlined core::ub_checks::check_language_ub) { + scope 9 (inlined core::ub_checks::check_language_ub::runtime) { } } - scope 8 (inlined std::ptr::from_raw_parts::) { - } } - scope 9 (inlined std::ptr::const_ptr::::wrapping_sub) { - scope 10 (inlined core::num::::wrapping_neg) { - scope 11 (inlined core::num::::wrapping_sub) { - } - } - scope 12 (inlined std::ptr::const_ptr::::wrapping_offset) { - } + scope 10 (inlined without_provenance_mut::) { } } - scope 13 (inlined as PartialEq>::eq) { + scope 5 (inlined std::ptr::const_ptr::::addr) { + scope 6 (inlined std::ptr::const_ptr::::cast::<()>) { + } + } + scope 11 (inlined as PartialEq>::eq) { let mut _5: *mut T; let mut _6: *mut T; - scope 14 (inlined NonNull::::as_ptr) { + scope 12 (inlined NonNull::::as_ptr) { } - scope 15 (inlined NonNull::::as_ptr) { + scope 13 (inlined NonNull::::as_ptr) { } } - scope 16 (inlined NonNull::::add) { + scope 14 (inlined NonNull::::add) { let mut _8: *const T; let mut _9: *const T; - scope 17 (inlined NonNull::::as_ptr) { + scope 15 (inlined NonNull::::as_ptr) { } } } @@ -55,6 +49,7 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { StorageLive(_2); StorageLive(_3); StorageLive(_11); + StorageLive(_12); StorageLive(_4); _2 = copy ((*_1).0: std::ptr::NonNull); _3 = copy ((*_1).1: *const T); @@ -92,42 +87,33 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { bb3: { _0 = const {transmute(0x0000000000000000): Option<&T>}; StorageDead(_7); - goto -> bb9; + goto -> bb8; } bb4: { - _11 = copy _3 as *const u8 (PtrToPtr); - StorageLive(_12); - _12 = Eq(copy _11, const {0x0 as *const u8}); - switchInt(move _12) -> [0: bb5, otherwise: bb8]; + _11 = copy _3 as usize (Transmute); + switchInt(copy _11) -> [0: bb5, otherwise: bb6]; } bb5: { - StorageDead(_12); - StorageLive(_13); - _13 = arith_offset::(move _11, const -1_isize) -> [return: bb6, unwind unreachable]; + _0 = const {transmute(0x0000000000000000): Option<&T>}; + goto -> bb8; } bb6: { - _14 = move _13 as *const T (PtrToPtr); - StorageDead(_13); - ((*_1).1: *const T) = copy _14; + _12 = SubUnchecked(copy _11, const 1_usize); + ((*_1).1: *const T) = copy _12 as *const T (Transmute); goto -> bb7; } bb7: { _0 = copy _2 as std::option::Option<&T> (Transmute); - goto -> bb9; + goto -> bb8; } bb8: { - _0 = const {transmute(0x0000000000000000): Option<&T>}; - StorageDead(_12); - goto -> bb9; - } - - bb9: { StorageDead(_4); + StorageDead(_12); StorageDead(_11); StorageDead(_3); StorageDead(_2); From 39118d6181df1ffef6e4407bbf273542ae5efe3c Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Thu, 13 Feb 2025 22:53:35 -0800 Subject: [PATCH 3/4] Go back to `Some` instead of transmuting to it. This adds a few more statements to `next`, but optimizes better in the loops (saving 2 blocks in `forward_loop`, for example) --- library/core/src/slice/iter/macros.rs | 2 +- ...ated_loop.PreCodegen.after.panic-abort.mir | 145 +++++++++--------- ...ward_loop.PreCodegen.after.panic-abort.mir | 92 +++++------ ...ard_loop.PreCodegen.after.panic-unwind.mir | 98 ++++++------ ...iter_next.PreCodegen.after.panic-abort.mir | 16 +- ...ter_next.PreCodegen.after.panic-unwind.mir | 16 +- 6 files changed, 205 insertions(+), 164 deletions(-) diff --git a/library/core/src/slice/iter/macros.rs b/library/core/src/slice/iter/macros.rs index 101f3157651f6..c4b8583b75fff 100644 --- a/library/core/src/slice/iter/macros.rs +++ b/library/core/src/slice/iter/macros.rs @@ -173,7 +173,7 @@ macro_rules! iterator { } self.ptr = ptr.add(1); } - crate::intrinsics::transmute::, Option<$elem>>(ptr) + Some({ptr}.$into_ref()) } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index 80b96e01041e6..7ef532d222dc3 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -7,27 +7,27 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let mut _11: std::ptr::NonNull; let mut _12: *const T; let mut _13: usize; - let mut _31: std::option::Option<(usize, &T)>; - let mut _34: &impl Fn(usize, &T); - let mut _35: (usize, &T); - let _36: (); + let mut _32: std::option::Option<(usize, &T)>; + let mut _35: &impl Fn(usize, &T); + let mut _36: (usize, &T); + let _37: (); scope 1 { debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _12; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; debug ((iter: Enumerate>).1: usize) => _13; - let _32: usize; - let _33: &T; + let _33: usize; + let _34: &T; scope 2 { - debug i => _32; - debug x => _33; + debug i => _33; + debug x => _34; } scope 19 (inlined > as Iterator>::next) { - let mut _25: std::option::Option<&T>; - let mut _29: (usize, bool); - let mut _30: (usize, &T); + let mut _27: std::option::Option<&T>; + let mut _30: (usize, bool); + let mut _31: (usize, &T); scope 20 { - let _28: usize; + let _29: usize; scope 25 { } } @@ -42,8 +42,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 26 (inlined as Try>::branch) { - let mut _26: isize; - let _27: &T; + let _28: &T; scope 27 { } } @@ -53,6 +52,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let mut _19: bool; let mut _22: std::ptr::NonNull; let mut _24: usize; + let _26: &T; scope 30 { let _15: *const T; scope 31 { @@ -85,6 +85,13 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { scope 43 (inlined NonNull::::as_ptr) { } } + scope 44 (inlined NonNull::::as_ref::<'_>) { + let _25: *const T; + scope 45 (inlined NonNull::::as_ptr) { + } + scope 46 (inlined std::ptr::mut_ptr::::cast_const) { + } + } } } } @@ -175,15 +182,16 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb4: { - StorageLive(_31); - StorageLive(_28); + StorageLive(_32); StorageLive(_29); - StorageLive(_25); + StorageLive(_30); + StorageLive(_27); StorageLive(_14); StorageLive(_15); StorageLive(_23); StorageLive(_24); StorageLive(_16); + StorageLive(_26); _14 = copy _11; _15 = copy _12; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; @@ -214,103 +222,96 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { StorageDead(_21); _11 = move _22; StorageDead(_22); - goto -> bb11; + goto -> bb13; } bb7: { StorageDead(_19); + StorageDead(_26); StorageDead(_16); StorageDead(_24); StorageDead(_23); StorageDead(_15); StorageDead(_14); - StorageLive(_26); - goto -> bb12; + goto -> bb10; } bb8: { _23 = copy _15 as usize (Transmute); - switchInt(copy _23) -> [0: bb9, otherwise: bb10]; + switchInt(copy _23) -> [0: bb9, otherwise: bb12]; } bb9: { + StorageDead(_26); StorageDead(_16); StorageDead(_24); StorageDead(_23); StorageDead(_15); StorageDead(_14); - StorageLive(_26); - goto -> bb12; + goto -> bb10; } bb10: { - _24 = SubUnchecked(copy _23, const 1_usize); - _12 = copy _24 as *const T (Transmute); - goto -> bb11; - } - - bb11: { - _25 = copy _14 as std::option::Option<&T> (Transmute); - StorageDead(_16); - StorageDead(_24); - StorageDead(_23); - StorageDead(_15); - StorageDead(_14); - StorageLive(_26); - _26 = discriminant(_25); - switchInt(move _26) -> [0: bb12, 1: bb14, otherwise: bb17]; - } - - bb12: { - StorageDead(_26); - StorageDead(_25); + StorageDead(_27); + StorageDead(_30); StorageDead(_29); - StorageDead(_28); - StorageDead(_31); + StorageDead(_32); StorageDead(_11); StorageDead(_12); StorageDead(_13); - drop(_2) -> [return: bb13, unwind unreachable]; + drop(_2) -> [return: bb11, unwind unreachable]; } - bb13: { + bb11: { return; } - bb14: { - _27 = move ((_25 as Some).0: &T); - StorageDead(_26); + bb12: { + _24 = SubUnchecked(copy _23, const 1_usize); + _12 = copy _24 as *const T (Transmute); + goto -> bb13; + } + + bb13: { + StorageLive(_25); + _25 = copy _14 as *const T (Transmute); + _26 = &(*_25); StorageDead(_25); - _28 = copy _13; - _29 = AddWithOverflow(copy _13, const 1_usize); - assert(!move (_29.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb15, unwind unreachable]; + _27 = Option::<&T>::Some(copy _26); + StorageDead(_26); + StorageDead(_16); + StorageDead(_24); + StorageDead(_23); + StorageDead(_15); + StorageDead(_14); + _28 = move ((_27 as Some).0: &T); + StorageDead(_27); + _29 = copy _13; + _30 = AddWithOverflow(copy _13, const 1_usize); + assert(!move (_30.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb14, unwind unreachable]; } - bb15: { - _13 = move (_29.0: usize); - StorageLive(_30); - _30 = (copy _28, copy _27); - _31 = Option::<(usize, &T)>::Some(move _30); + bb14: { + _13 = move (_30.0: usize); + StorageLive(_31); + _31 = (copy _29, copy _28); + _32 = Option::<(usize, &T)>::Some(move _31); + StorageDead(_31); StorageDead(_30); StorageDead(_29); - StorageDead(_28); - _32 = copy (((_31 as Some).0: (usize, &T)).0: usize); - _33 = copy (((_31 as Some).0: (usize, &T)).1: &T); - StorageLive(_34); - _34 = &_2; + _33 = copy (((_32 as Some).0: (usize, &T)).0: usize); + _34 = copy (((_32 as Some).0: (usize, &T)).1: &T); StorageLive(_35); - _35 = (copy _32, copy _33); - _36 = >::call(move _34, move _35) -> [return: bb16, unwind unreachable]; + _35 = &_2; + StorageLive(_36); + _36 = (copy _33, copy _34); + _37 = >::call(move _35, move _36) -> [return: bb15, unwind unreachable]; } - bb16: { + bb15: { + StorageDead(_36); StorageDead(_35); - StorageDead(_34); - StorageDead(_31); + StorageDead(_32); goto -> bb4; } - - bb17: { - unreachable; - } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir index c539170055d66..62787f3447cee 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -6,18 +6,17 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _0: (); let mut _11: std::ptr::NonNull; let mut _12: *const T; - let mut _24: std::option::Option<&T>; - let mut _25: isize; - let mut _27: &impl Fn(&T); - let mut _28: (&T,); - let _29: (); + let mut _26: std::option::Option<&T>; + let mut _28: &impl Fn(&T); + let mut _29: (&T,); + let _30: (); scope 1 { debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12; debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - let _26: &T; + let _27: &T; scope 2 { - debug x => _26; + debug x => _27; } scope 17 (inlined as Iterator>::next) { let _13: std::ptr::NonNull; @@ -25,6 +24,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _18: bool; let mut _21: std::ptr::NonNull; let mut _23: usize; + let _25: &T; scope 18 { let _14: *const T; scope 19 { @@ -57,6 +57,13 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { scope 31 (inlined NonNull::::as_ptr) { } } + scope 32 (inlined NonNull::::as_ref::<'_>) { + let _24: *const T; + scope 33 (inlined NonNull::::as_ptr) { + } + scope 34 (inlined std::ptr::mut_ptr::::cast_const) { + } + } } } } @@ -140,12 +147,13 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_24); + StorageLive(_26); StorageLive(_13); StorageLive(_14); StorageLive(_22); StorageLive(_23); StorageLive(_15); + StorageLive(_25); _13 = copy _11; _14 = copy _12; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; @@ -176,78 +184,76 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { StorageDead(_20); _11 = move _21; StorageDead(_21); - goto -> bb11; + goto -> bb13; } bb7: { StorageDead(_18); + StorageDead(_25); StorageDead(_15); StorageDead(_23); StorageDead(_22); StorageDead(_14); StorageDead(_13); - goto -> bb12; + goto -> bb10; } bb8: { _22 = copy _14 as usize (Transmute); - switchInt(copy _22) -> [0: bb9, otherwise: bb10]; + switchInt(copy _22) -> [0: bb9, otherwise: bb12]; } bb9: { + StorageDead(_25); StorageDead(_15); StorageDead(_23); StorageDead(_22); StorageDead(_14); StorageDead(_13); - goto -> bb12; + goto -> bb10; } bb10: { - _23 = SubUnchecked(copy _22, const 1_usize); - _12 = copy _23 as *const T (Transmute); - goto -> bb11; + StorageDead(_26); + StorageDead(_11); + StorageDead(_12); + drop(_2) -> [return: bb11, unwind unreachable]; } bb11: { - _24 = copy _13 as std::option::Option<&T> (Transmute); - StorageDead(_15); - StorageDead(_23); - StorageDead(_22); - StorageDead(_14); - StorageDead(_13); - _25 = discriminant(_24); - switchInt(move _25) -> [0: bb12, 1: bb14, otherwise: bb16]; + return; } bb12: { - StorageDead(_24); - StorageDead(_11); - StorageDead(_12); - drop(_2) -> [return: bb13, unwind unreachable]; + _23 = SubUnchecked(copy _22, const 1_usize); + _12 = copy _23 as *const T (Transmute); + goto -> bb13; } bb13: { - return; - } - - bb14: { - _26 = copy ((_24 as Some).0: &T); - StorageLive(_27); - _27 = &_2; + StorageLive(_24); + _24 = copy _13 as *const T (Transmute); + _25 = &(*_24); + StorageDead(_24); + _26 = Option::<&T>::Some(copy _25); + StorageDead(_25); + StorageDead(_15); + StorageDead(_23); + StorageDead(_22); + StorageDead(_14); + StorageDead(_13); + _27 = copy ((_26 as Some).0: &T); StorageLive(_28); - _28 = (copy _26,); - _29 = >::call(move _27, move _28) -> [return: bb15, unwind unreachable]; + _28 = &_2; + StorageLive(_29); + _29 = (copy _27,); + _30 = >::call(move _28, move _29) -> [return: bb14, unwind unreachable]; } - bb15: { + bb14: { + StorageDead(_29); StorageDead(_28); - StorageDead(_27); - StorageDead(_24); + StorageDead(_26); goto -> bb4; } - - bb16: { - unreachable; - } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir index dc537ad87402f..e5478e2791888 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -6,18 +6,17 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _0: (); let mut _11: std::ptr::NonNull; let mut _12: *const T; - let mut _24: std::option::Option<&T>; - let mut _25: isize; - let mut _27: &impl Fn(&T); - let mut _28: (&T,); - let _29: (); + let mut _26: std::option::Option<&T>; + let mut _28: &impl Fn(&T); + let mut _29: (&T,); + let _30: (); scope 1 { debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12; debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - let _26: &T; + let _27: &T; scope 2 { - debug x => _26; + debug x => _27; } scope 17 (inlined as Iterator>::next) { let _13: std::ptr::NonNull; @@ -25,6 +24,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _18: bool; let mut _21: std::ptr::NonNull; let mut _23: usize; + let _25: &T; scope 18 { let _14: *const T; scope 19 { @@ -57,6 +57,13 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { scope 31 (inlined NonNull::::as_ptr) { } } + scope 32 (inlined NonNull::::as_ref::<'_>) { + let _24: *const T; + scope 33 (inlined NonNull::::as_ptr) { + } + scope 34 (inlined std::ptr::mut_ptr::::cast_const) { + } + } } } } @@ -140,12 +147,13 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_24); + StorageLive(_26); StorageLive(_13); StorageLive(_14); StorageLive(_22); StorageLive(_23); StorageLive(_15); + StorageLive(_25); _13 = copy _11; _14 = copy _12; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; @@ -176,86 +184,84 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { StorageDead(_20); _11 = move _21; StorageDead(_21); - goto -> bb11; + goto -> bb13; } bb7: { StorageDead(_18); + StorageDead(_25); StorageDead(_15); StorageDead(_23); StorageDead(_22); StorageDead(_14); StorageDead(_13); - goto -> bb12; + goto -> bb10; } bb8: { _22 = copy _14 as usize (Transmute); - switchInt(copy _22) -> [0: bb9, otherwise: bb10]; + switchInt(copy _22) -> [0: bb9, otherwise: bb12]; } bb9: { + StorageDead(_25); StorageDead(_15); StorageDead(_23); StorageDead(_22); StorageDead(_14); StorageDead(_13); - goto -> bb12; + goto -> bb10; } bb10: { - _23 = SubUnchecked(copy _22, const 1_usize); - _12 = copy _23 as *const T (Transmute); - goto -> bb11; + StorageDead(_26); + StorageDead(_11); + StorageDead(_12); + drop(_2) -> [return: bb11, unwind continue]; } bb11: { - _24 = copy _13 as std::option::Option<&T> (Transmute); - StorageDead(_15); - StorageDead(_23); - StorageDead(_22); - StorageDead(_14); - StorageDead(_13); - _25 = discriminant(_24); - switchInt(move _25) -> [0: bb12, 1: bb14, otherwise: bb18]; + return; } bb12: { - StorageDead(_24); - StorageDead(_11); - StorageDead(_12); - drop(_2) -> [return: bb13, unwind continue]; + _23 = SubUnchecked(copy _22, const 1_usize); + _12 = copy _23 as *const T (Transmute); + goto -> bb13; } bb13: { - return; - } - - bb14: { - _26 = copy ((_24 as Some).0: &T); - StorageLive(_27); - _27 = &_2; + StorageLive(_24); + _24 = copy _13 as *const T (Transmute); + _25 = &(*_24); + StorageDead(_24); + _26 = Option::<&T>::Some(copy _25); + StorageDead(_25); + StorageDead(_15); + StorageDead(_23); + StorageDead(_22); + StorageDead(_14); + StorageDead(_13); + _27 = copy ((_26 as Some).0: &T); StorageLive(_28); - _28 = (copy _26,); - _29 = >::call(move _27, move _28) -> [return: bb15, unwind: bb16]; + _28 = &_2; + StorageLive(_29); + _29 = (copy _27,); + _30 = >::call(move _28, move _29) -> [return: bb14, unwind: bb15]; } - bb15: { + bb14: { + StorageDead(_29); StorageDead(_28); - StorageDead(_27); - StorageDead(_24); + StorageDead(_26); goto -> bb4; } - bb16 (cleanup): { - drop(_2) -> [return: bb17, unwind terminate(cleanup)]; + bb15 (cleanup): { + drop(_2) -> [return: bb16, unwind terminate(cleanup)]; } - bb17 (cleanup): { + bb16 (cleanup): { resume; } - - bb18: { - unreachable; - } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir index 0fe469c21f742..b6df2300efb1f 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir @@ -9,6 +9,7 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { let mut _7: bool; let mut _10: std::ptr::NonNull; let mut _12: usize; + let _14: &T; scope 2 { let _3: *const T; scope 3 { @@ -41,6 +42,13 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { scope 15 (inlined NonNull::::as_ptr) { } } + scope 16 (inlined NonNull::::as_ref::<'_>) { + let _13: *const T; + scope 17 (inlined NonNull::::as_ptr) { + } + scope 18 (inlined std::ptr::mut_ptr::::cast_const) { + } + } } } } @@ -51,6 +59,7 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { StorageLive(_11); StorageLive(_12); StorageLive(_4); + StorageLive(_14); _2 = copy ((*_1).0: std::ptr::NonNull); _3 = copy ((*_1).1: *const T); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb4]; @@ -107,11 +116,16 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { } bb7: { - _0 = copy _2 as std::option::Option<&T> (Transmute); + StorageLive(_13); + _13 = copy _2 as *const T (Transmute); + _14 = &(*_13); + StorageDead(_13); + _0 = Option::<&T>::Some(copy _14); goto -> bb8; } bb8: { + StorageDead(_14); StorageDead(_4); StorageDead(_12); StorageDead(_11); diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir index 0fe469c21f742..b6df2300efb1f 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir @@ -9,6 +9,7 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { let mut _7: bool; let mut _10: std::ptr::NonNull; let mut _12: usize; + let _14: &T; scope 2 { let _3: *const T; scope 3 { @@ -41,6 +42,13 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { scope 15 (inlined NonNull::::as_ptr) { } } + scope 16 (inlined NonNull::::as_ref::<'_>) { + let _13: *const T; + scope 17 (inlined NonNull::::as_ptr) { + } + scope 18 (inlined std::ptr::mut_ptr::::cast_const) { + } + } } } } @@ -51,6 +59,7 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { StorageLive(_11); StorageLive(_12); StorageLive(_4); + StorageLive(_14); _2 = copy ((*_1).0: std::ptr::NonNull); _3 = copy ((*_1).1: *const T); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb4]; @@ -107,11 +116,16 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> { } bb7: { - _0 = copy _2 as std::option::Option<&T> (Transmute); + StorageLive(_13); + _13 = copy _2 as *const T (Transmute); + _14 = &(*_13); + StorageDead(_13); + _0 = Option::<&T>::Some(copy _14); goto -> bb8; } bb8: { + StorageDead(_14); StorageDead(_4); StorageDead(_12); StorageDead(_11); From 7add35831955f8fc5ba08dcdb93921cd8de17841 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Fri, 14 Feb 2025 22:59:19 -0800 Subject: [PATCH 4/4] Add real safety comments --- library/core/src/slice/iter/macros.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/library/core/src/slice/iter/macros.rs b/library/core/src/slice/iter/macros.rs index c4b8583b75fff..b1456a1bc1da8 100644 --- a/library/core/src/slice/iter/macros.rs +++ b/library/core/src/slice/iter/macros.rs @@ -159,20 +159,33 @@ macro_rules! iterator { let ptr = self.ptr; let end_or_len = self.end_or_len; - // SAFETY: Type invariants. + // SAFETY: See inner comments. (For some reason having multiple + // block breaks inlining this -- if you can fix that please do!) unsafe { if T::IS_ZST { let len = end_or_len.addr(); if len == 0 { return None; } + // SAFETY: just checked that it's not zero, so subtracting one + // cannot wrap. (Ideally this would be `checked_sub`, which + // does the same thing internally, but as of 2025-02 that + // doesn't optimize quite as small in MIR.) self.end_or_len = without_provenance_mut(len.unchecked_sub(1)); } else { + // SAFETY: by type invariant, the `end_or_len` field is always + // non-null for a non-ZST pointee. (This transmute ensures we + // get `!nonnull` metadata on the load of the field.) if ptr == crate::intrinsics::transmute::<$ptr, NonNull>(end_or_len) { return None; } + // SAFETY: since it's not empty, per the check above, moving + // forward one keeps us inside the slice, and this is valid. self.ptr = ptr.add(1); } + // SAFETY: Now that we know it wasn't empty and we've moved past + // the first one (to avoid giving a duplicate `&mut` next time), + // we can give out a reference to it. Some({ptr}.$into_ref()) } }