@@ -3020,6 +3020,28 @@ macro_rules! iterator {
3020
3020
{ $( $mut_: tt ) * } ,
3021
3021
{ $( $extra: tt) * }
3022
3022
) => {
3023
+ // Returns the first element and moves the start of the iterator forwards by 1.
3024
+ // Greatly improves performance compared to an inlined function. The iterator
3025
+ // must not be empty.
3026
+ macro_rules! next_unchecked {
3027
+ ( $self: ident) => { & $( $mut_ ) * * $self. post_inc_start( 1 ) }
3028
+ }
3029
+
3030
+ // Returns the last element and moves the end of the iterator backwards by 1.
3031
+ // Greatly improves performance compared to an inlined function. The iterator
3032
+ // must not be empty.
3033
+ macro_rules! next_back_unchecked {
3034
+ ( $self: ident) => { & $( $mut_ ) * * $self. pre_dec_end( 1 ) }
3035
+ }
3036
+
3037
+ // Shrinks the iterator when T is a ZST, by moving the end of the iterator
3038
+ // backwards by `n`. `n` must not exceed `self.len()`.
3039
+ macro_rules! zst_shrink {
3040
+ ( $self: ident, $n: ident) => {
3041
+ $self. end = ( $self. end as * $raw_mut u8 ) . wrapping_offset( -$n) as * $raw_mut T ;
3042
+ }
3043
+ }
3044
+
3023
3045
impl <' a, T > $name<' a, T > {
3024
3046
// Helper function for creating a slice from the iterator.
3025
3047
#[ inline( always) ]
@@ -3029,12 +3051,11 @@ macro_rules! iterator {
3029
3051
3030
3052
// Helper function for moving the start of the iterator forwards by `offset` elements,
3031
3053
// returning the old start.
3032
- // Unsafe because the offset must be in-bounds or one-past-the-end .
3054
+ // Unsafe because the offset must not exceed `self.len()` .
3033
3055
#[ inline( always) ]
3034
3056
unsafe fn post_inc_start( & mut self , offset: isize ) -> * $raw_mut T {
3035
3057
if mem:: size_of:: <T >( ) == 0 {
3036
- // This is *reducing* the length. `ptr` never changes with ZST.
3037
- self . end = ( self . end as * $raw_mut u8 ) . wrapping_offset( -offset) as * $raw_mut T ;
3058
+ zst_shrink!( self , offset) ;
3038
3059
self . ptr
3039
3060
} else {
3040
3061
let old = self . ptr;
@@ -3045,11 +3066,11 @@ macro_rules! iterator {
3045
3066
3046
3067
// Helper function for moving the end of the iterator backwards by `offset` elements,
3047
3068
// returning the new end.
3048
- // Unsafe because the offset must be in-bounds or one-past-the-end .
3069
+ // Unsafe because the offset must not exceed `self.len()` .
3049
3070
#[ inline( always) ]
3050
3071
unsafe fn pre_dec_end( & mut self , offset: isize ) -> * $raw_mut T {
3051
3072
if mem:: size_of:: <T >( ) == 0 {
3052
- self . end = ( self . end as * $raw_mut u8 ) . wrapping_offset ( - offset) as * $raw_mut T ;
3073
+ zst_shrink! ( self , offset) ;
3053
3074
self . ptr
3054
3075
} else {
3055
3076
self . end = self . end. offset( -offset) ;
@@ -3086,7 +3107,7 @@ macro_rules! iterator {
3086
3107
if is_empty!( self ) {
3087
3108
None
3088
3109
} else {
3089
- Some ( & $ ( $mut_ ) * * self . post_inc_start ( 1 ) )
3110
+ Some ( next_unchecked! ( self ) )
3090
3111
}
3091
3112
}
3092
3113
}
@@ -3115,11 +3136,10 @@ macro_rules! iterator {
3115
3136
}
3116
3137
return None ;
3117
3138
}
3118
- // We are in bounds. `offset ` does the right thing even for ZSTs.
3139
+ // We are in bounds. `post_inc_start ` does the right thing even for ZSTs.
3119
3140
unsafe {
3120
- let elem = Some ( & $( $mut_ ) * * self . ptr. add( n) ) ;
3121
- self . post_inc_start( ( n as isize ) . wrapping_add( 1 ) ) ;
3122
- elem
3141
+ self . post_inc_start( n as isize ) ;
3142
+ Some ( next_unchecked!( self ) )
3123
3143
}
3124
3144
}
3125
3145
@@ -3136,13 +3156,13 @@ macro_rules! iterator {
3136
3156
let mut accum = init;
3137
3157
unsafe {
3138
3158
while len!( self ) >= 4 {
3139
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3140
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3141
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3142
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3159
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3160
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3161
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3162
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3143
3163
}
3144
3164
while !is_empty!( self ) {
3145
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3165
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3146
3166
}
3147
3167
}
3148
3168
Try :: from_ok( accum)
@@ -3213,11 +3233,25 @@ macro_rules! iterator {
3213
3233
if is_empty!( self ) {
3214
3234
None
3215
3235
} else {
3216
- Some ( & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) )
3236
+ Some ( next_back_unchecked! ( self ) )
3217
3237
}
3218
3238
}
3219
3239
}
3220
3240
3241
+ #[ inline]
3242
+ fn nth_back( & mut self , n: usize ) -> Option <$elem> {
3243
+ if n >= len!( self ) {
3244
+ // This iterator is now empty.
3245
+ self . end = self . ptr;
3246
+ return None ;
3247
+ }
3248
+ // We are in bounds. `pre_dec_end` does the right thing even for ZSTs.
3249
+ unsafe {
3250
+ self . pre_dec_end( n as isize ) ;
3251
+ Some ( next_back_unchecked!( self ) )
3252
+ }
3253
+ }
3254
+
3221
3255
#[ inline]
3222
3256
fn try_rfold<B , F , R >( & mut self , init: B , mut f: F ) -> R where
3223
3257
Self : Sized , F : FnMut ( B , Self :: Item ) -> R , R : Try <Ok =B >
@@ -3226,14 +3260,14 @@ macro_rules! iterator {
3226
3260
let mut accum = init;
3227
3261
unsafe {
3228
3262
while len!( self ) >= 4 {
3229
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3230
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3231
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3232
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3263
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3264
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3265
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3266
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3233
3267
}
3234
3268
// inlining is_empty everywhere makes a huge performance difference
3235
3269
while !is_empty!( self ) {
3236
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3270
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3237
3271
}
3238
3272
}
3239
3273
Try :: from_ok( accum)
0 commit comments