Skip to content

Commit e6fc62a

Browse files
committed
Don't use SIMD in mem::swap for types smaller than the block size
LLVM isn't able to remove the alloca for the unaligned block in the SIMD tail in some cases, so doing this helps SRoA work in cases where it currently doesn't. Found in the `replace_with` RFC discussion.
1 parent 874dec2 commit e6fc62a

File tree

3 files changed

+40
-1
lines changed

3 files changed

+40
-1
lines changed

src/libcore/mem.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -638,7 +638,7 @@ pub unsafe fn uninitialized<T>() -> T {
638638
#[stable(feature = "rust1", since = "1.0.0")]
639639
pub fn swap<T>(x: &mut T, y: &mut T) {
640640
unsafe {
641-
ptr::swap_nonoverlapping(x, y, 1);
641+
ptr::swap_nonoverlapping_one(x, y);
642642
}
643643
}
644644

src/libcore/ptr.rs

+13
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,19 @@ pub unsafe fn swap_nonoverlapping<T>(x: *mut T, y: *mut T, count: usize) {
187187
swap_nonoverlapping_bytes(x, y, len)
188188
}
189189

190+
#[inline]
191+
pub(crate) unsafe fn swap_nonoverlapping_one<T>(x: *mut T, y: *mut T) {
192+
// For types smaller than the block optimization below,
193+
// just swap directly to avoid pessimizing codegen.
194+
if mem::size_of::<T>() < 32 {
195+
let z = read(x);
196+
copy_nonoverlapping(y, x, 1);
197+
write(y, z);
198+
} else {
199+
swap_nonoverlapping(x, y, 1);
200+
}
201+
}
202+
190203
#[inline]
191204
unsafe fn swap_nonoverlapping_bytes(x: *mut u8, y: *mut u8, len: usize) {
192205
// The approach here is to utilize simd to swap x & y efficiently. Testing reveals

src/test/codegen/swap-small-types.rs

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
// compile-flags: -O
12+
13+
#![crate_type = "lib"]
14+
15+
use std::mem::swap;
16+
17+
type RGB48 = [u16; 3];
18+
19+
// CHECK-LABEL: @swap_rgb48
20+
#[no_mangle]
21+
pub fn swap_rgb48(x: &mut RGB48, y: &mut RGB48) {
22+
// CHECK-NOT: alloca
23+
// CHECK: load i48
24+
// CHECK: store i48
25+
swap(x, y)
26+
}

0 commit comments

Comments
 (0)