Skip to content

Commit 05b5e01

Browse files
cesarbcesarb
committed
Finish porting code to "SIMD groundwork part 1"
Finish porting the SIMD code to rust-lang/rust#27169
1 parent 9e5a416 commit 05b5e01

File tree

5 files changed

+199
-110
lines changed

5 files changed

+199
-110
lines changed

‎src/blake2.rs

+6-6
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,9 @@ macro_rules! blake2_impl {
106106
const IV: [$word; 8] = $IV;
107107

108108
#[inline(always)]
109-
fn iv0() -> $vec { $vec(IV[0], IV[1], IV[2], IV[3]) }
109+
fn iv0() -> $vec { $vec::new(IV[0], IV[1], IV[2], IV[3]) }
110110
#[inline(always)]
111-
fn iv1() -> $vec { $vec(IV[4], IV[5], IV[6], IV[7]) }
111+
fn iv1() -> $vec { $vec::new(IV[4], IV[5], IV[6], IV[7]) }
112112

113113
/// Convenience function for all-in-one computation.
114114
pub fn $func(nn: usize, k: &[u8], data: &[u8]) -> $result {
@@ -129,7 +129,7 @@ macro_rules! blake2_impl {
129129
let p0 = 0x01010000 ^ ((kk as $word) << 8) ^ (nn as $word);
130130
let mut state = $state {
131131
m: [0; 16],
132-
h: [iv0() ^ $vec(p0, 0, 0, 0), iv1()],
132+
h: [iv0() ^ $vec::new(p0, 0, 0, 0), iv1()],
133133
t: 0,
134134
nn: nn,
135135
};
@@ -145,8 +145,8 @@ macro_rules! blake2_impl {
145145
pub fn with_parameter_block(p: &[$word; 8]) -> Self {
146146
$state {
147147
m: [0; 16],
148-
h: [iv0() ^ $vec(p[0], p[1], p[2], p[3]),
149-
iv1() ^ $vec(p[4], p[5], p[6], p[7])],
148+
h: [iv0() ^ $vec::new(p[0], p[1], p[2], p[3]),
149+
iv1() ^ $vec::new(p[4], p[5], p[6], p[7])],
150150
t: 0,
151151
nn: p[0] as u8 as usize,
152152
}
@@ -263,7 +263,7 @@ macro_rules! blake2_impl {
263263
h[0],
264264
h[1],
265265
iv0(),
266-
iv1() ^ $vec(t0, t1, f0, f1),
266+
iv1() ^ $vec::new(t0, t1, f0, f1),
267267
];
268268

269269
$state::round(&mut v, m, &SIGMA[0]);

‎src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ extern crate constant_time_eq;
3737

3838
mod as_bytes;
3939
mod bytes;
40+
4041
mod simdty;
42+
#[cfg(feature = "simd")] mod simdint;
4143
mod simd;
4244

4345
#[macro_use]

‎src/simd.rs

+88-81
Original file line numberDiff line numberDiff line change
@@ -24,33 +24,8 @@
2424
// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
2525
// DEALINGS IN THE SOFTWARE.
2626

27-
#[cfg(feature = "simd_opt")]
28-
use std::mem::transmute;
29-
3027
pub use simdty::{u32x4, u64x4};
31-
32-
#[cfg(feature = "simd")]
33-
extern "platform-intrinsic" {
34-
fn simd_add<T>(x: T, y: T) -> T;
35-
fn simd_shl<T>(x: T, y: T) -> T;
36-
fn simd_shr<T>(x: T, y: T) -> T;
37-
fn simd_xor<T>(x: T, y: T) -> T;
38-
}
39-
40-
#[cfg(feature = "simd_opt")]
41-
extern "platform-intrinsic" {
42-
fn simd_shuffle8<T, Elem>(v: T, w: T,
43-
i0: u32, i1: u32, i2: u32, i3: u32,
44-
i4: u32, i5: u32, i6: u32, i7: u32) -> T;
45-
46-
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
47-
fn simd_shuffle16<T, Elem>(v: T, w: T,
48-
i0: u32, i1: u32, i2: u32, i3: u32,
49-
i4: u32, i5: u32, i6: u32, i7: u32,
50-
i8: u32, i9: u32, i10: u32, i11: u32,
51-
i12: u32, i13: u32, i14: u32, i15: u32,
52-
) -> T;
53-
}
28+
#[cfg(feature = "simd")] use simdint;
5429

5530
use std::ops::{Add, BitXor, Shl, Shr};
5631

@@ -62,16 +37,16 @@ macro_rules! impl_ops {
6237
#[cfg(feature = "simd")]
6338
#[inline(always)]
6439
fn add(self, rhs: Self) -> Self::Output {
65-
unsafe { simd_add(self, rhs) }
40+
unsafe { simdint::simd_add(self, rhs) }
6641
}
6742

6843
#[cfg(not(feature = "simd"))]
6944
#[inline(always)]
7045
fn add(self, rhs: Self) -> Self::Output {
71-
$vec(self.0.wrapping_add(rhs.0),
72-
self.1.wrapping_add(rhs.1),
73-
self.2.wrapping_add(rhs.2),
74-
self.3.wrapping_add(rhs.3))
46+
$vec::new(self.0.wrapping_add(rhs.0),
47+
self.1.wrapping_add(rhs.1),
48+
self.2.wrapping_add(rhs.2),
49+
self.3.wrapping_add(rhs.3))
7550
}
7651
}
7752

@@ -81,16 +56,16 @@ macro_rules! impl_ops {
8156
#[cfg(feature = "simd")]
8257
#[inline(always)]
8358
fn bitxor(self, rhs: Self) -> Self::Output {
84-
unsafe { simd_xor(self, rhs) }
59+
unsafe { simdint::simd_xor(self, rhs) }
8560
}
8661

8762
#[cfg(not(feature = "simd"))]
8863
#[inline(always)]
8964
fn bitxor(self, rhs: Self) -> Self::Output {
90-
$vec(self.0 ^ rhs.0,
91-
self.1 ^ rhs.1,
92-
self.2 ^ rhs.2,
93-
self.3 ^ rhs.3)
65+
$vec::new(self.0 ^ rhs.0,
66+
self.1 ^ rhs.1,
67+
self.2 ^ rhs.2,
68+
self.3 ^ rhs.3)
9469
}
9570
}
9671

@@ -100,16 +75,16 @@ macro_rules! impl_ops {
10075
#[cfg(feature = "simd")]
10176
#[inline(always)]
10277
fn shl(self, rhs: Self) -> Self::Output {
103-
unsafe { simd_shl(self, rhs) }
78+
unsafe { simdint::simd_shl(self, rhs) }
10479
}
10580

10681
#[cfg(not(feature = "simd"))]
10782
#[inline(always)]
10883
fn shl(self, rhs: Self) -> Self::Output {
109-
$vec(self.0 << rhs.0,
110-
self.1 << rhs.1,
111-
self.2 << rhs.2,
112-
self.3 << rhs.3)
84+
$vec::new(self.0 << rhs.0,
85+
self.1 << rhs.1,
86+
self.2 << rhs.2,
87+
self.3 << rhs.3)
11388
}
11489
}
11590

@@ -119,16 +94,16 @@ macro_rules! impl_ops {
11994
#[cfg(feature = "simd")]
12095
#[inline(always)]
12196
fn shr(self, rhs: Self) -> Self::Output {
122-
unsafe { simd_shr(self, rhs) }
97+
unsafe { simdint::simd_shr(self, rhs) }
12398
}
12499

125100
#[cfg(not(feature = "simd"))]
126101
#[inline(always)]
127102
fn shr(self, rhs: Self) -> Self::Output {
128-
$vec(self.0 >> rhs.0,
129-
self.1 >> rhs.1,
130-
self.2 >> rhs.2,
131-
self.3 >> rhs.3)
103+
$vec::new(self.0 >> rhs.0,
104+
self.1 >> rhs.1,
105+
self.2 >> rhs.2,
106+
self.3 >> rhs.3)
132107
}
133108
}
134109
}
@@ -163,7 +138,7 @@ macro_rules! impl_vector4_common {
163138
#[inline(always)]
164139
fn gather(src: &[$word], i0: usize, i1: usize,
165140
i2: usize, i3: usize) -> Self {
166-
$vec(src[i0], src[i1], src[i2], src[i3])
141+
$vec::new(src[i0], src[i1], src[i2], src[i3])
167142
}
168143

169144
#[cfg(target_endian = "little")]
@@ -173,10 +148,10 @@ macro_rules! impl_vector4_common {
173148
#[cfg(not(target_endian = "little"))]
174149
#[inline(always)]
175150
fn from_le(self) -> Self {
176-
$vec($word::from_le(self.0),
177-
$word::from_le(self.1),
178-
$word::from_le(self.2),
179-
$word::from_le(self.3))
151+
$vec::new($word::from_le(self.0),
152+
$word::from_le(self.1),
153+
$word::from_le(self.2),
154+
$word::from_le(self.3))
180155
}
181156

182157
#[cfg(target_endian = "little")]
@@ -186,10 +161,10 @@ macro_rules! impl_vector4_common {
186161
#[cfg(not(target_endian = "little"))]
187162
#[inline(always)]
188163
fn to_le(self) -> Self {
189-
$vec(self.0.to_le(),
190-
self.1.to_le(),
191-
self.2.to_le(),
192-
self.3.to_le())
164+
$vec::new(self.0.to_le(),
165+
self.1.to_le(),
166+
self.2.to_le(),
167+
self.3.to_le())
193168
}
194169

195170
#[inline(always)]
@@ -201,31 +176,52 @@ macro_rules! impl_vector4_common {
201176
let r = n as $word;
202177
let l = $bits - r;
203178

204-
(self >> $vec(r, r, r, r)) ^ (self << $vec(l, l, l, l))
179+
(self >> $vec::new(r, r, r, r)) ^ (self << $vec::new(l, l, l, l))
205180
}
206181

207182
#[cfg(not(feature = "simd"))]
208183
#[inline(always)]
209184
fn rotate_right_any(self, n: u32) -> Self {
210-
$vec(self.0.rotate_right(n),
211-
self.1.rotate_right(n),
212-
self.2.rotate_right(n),
213-
self.3.rotate_right(n))
185+
$vec::new(self.0.rotate_right(n),
186+
self.1.rotate_right(n),
187+
self.2.rotate_right(n),
188+
self.3.rotate_right(n))
189+
}
190+
191+
#[cfg(feature = "simd")]
192+
#[inline(always)]
193+
fn shuffle_left_1(self) -> Self {
194+
unsafe { simdint::simd_shuffle4(self, self, 1, 2, 3, 0) }
214195
}
215196

197+
#[cfg(not(feature = "simd"))]
216198
#[inline(always)]
217199
fn shuffle_left_1(self) -> Self {
218-
$vec(self.1, self.2, self.3, self.0)
200+
$vec::new(self.1, self.2, self.3, self.0)
201+
}
202+
203+
#[cfg(feature = "simd")]
204+
#[inline(always)]
205+
fn shuffle_left_2(self) -> Self {
206+
unsafe { simdint::simd_shuffle4(self, self, 2, 3, 0, 1) }
219207
}
220208

209+
#[cfg(not(feature = "simd"))]
221210
#[inline(always)]
222211
fn shuffle_left_2(self) -> Self {
223-
$vec(self.2, self.3, self.0, self.1)
212+
$vec::new(self.2, self.3, self.0, self.1)
213+
}
214+
215+
#[cfg(feature = "simd")]
216+
#[inline(always)]
217+
fn shuffle_left_3(self) -> Self {
218+
unsafe { simdint::simd_shuffle4(self, self, 3, 0, 1, 2) }
224219
}
225220

221+
#[cfg(not(feature = "simd"))]
226222
#[inline(always)]
227223
fn shuffle_left_3(self) -> Self {
228-
$vec(self.3, self.0, self.1, self.2)
224+
$vec::new(self.3, self.0, self.1, self.2)
229225
}
230226
}
231227
}
@@ -236,13 +232,16 @@ macro_rules! impl_vector4_common {
236232
#[inline(always)]
237233
fn u32x4_rotate_right_16(vec: u32x4) -> u32x4 {
238234
use simdty::u16x8;
235+
use std::mem::transmute;
236+
239237
unsafe {
240238
let tmp: u16x8 = transmute(vec);
241-
transmute(simd_shuffle8::<u16x8, u16>(tmp, tmp,
242-
1, 0,
243-
3, 2,
244-
5, 4,
245-
7, 6))
239+
let tmp: u16x8 = simdint::simd_shuffle8(tmp, tmp,
240+
1, 0,
241+
3, 2,
242+
5, 4,
243+
7, 6);
244+
transmute(tmp)
246245
}
247246
}
248247

@@ -268,13 +267,16 @@ impl Vector4<u32> for u32x4 {
268267
#[inline(always)]
269268
fn u64x4_rotate_right_32(vec: u64x4) -> u64x4 {
270269
use simdty::u32x8;
270+
use std::mem::transmute;
271+
271272
unsafe {
272273
let tmp: u32x8 = transmute(vec);
273-
transmute(simd_shuffle8::<u32x8, u32>(tmp, tmp,
274-
1, 0,
275-
3, 2,
276-
5, 4,
277-
7, 6))
274+
let tmp: u32x8 = simdint::simd_shuffle8(tmp, tmp,
275+
1, 0,
276+
3, 2,
277+
5, 4,
278+
7, 6);
279+
transmute(tmp)
278280
}
279281
}
280282

@@ -283,13 +285,16 @@ fn u64x4_rotate_right_32(vec: u64x4) -> u64x4 {
283285
#[inline(always)]
284286
fn u64x4_rotate_right_16(vec: u64x4) -> u64x4 {
285287
use simdty::u16x16;
288+
use std::mem::transmute;
289+
286290
unsafe {
287291
let tmp: u16x16 = transmute(vec);
288-
transmute(simd_shuffle16::<u16x16, u16>(tmp, tmp,
289-
1, 2, 3, 0,
290-
5, 6, 7, 4,
291-
9, 10, 11, 8,
292-
13, 14, 15, 12))
292+
let tmp: u16x16 = simdint::simd_shuffle16(tmp, tmp,
293+
1, 2, 3, 0,
294+
5, 6, 7, 4,
295+
9, 10, 11, 8,
296+
13, 14, 15, 12);
297+
transmute(tmp)
293298
}
294299
}
295300

@@ -314,9 +319,11 @@ fn vext_u64_u8(a: u64x2, b: u8) -> u64x2 {
314319
#[cfg(target_arch = "arm")]
315320
#[inline(always)]
316321
fn u64x4_rotate_right_u8(vec: u64x4, n: u8) -> u64x4 {
317-
let tmp0 = vext_u64_u8(u64x2(vec.0, vec.1), n);
318-
let tmp1 = vext_u64_u8(u64x2(vec.2, vec.3), n);
319-
u64x4(tmp0.0, tmp0.1, tmp1.0, tmp1.1)
322+
unsafe {
323+
let tmp0 = vext_u64_u8(simdint::simd_shuffle2(vec, vec, 0, 1), n);
324+
let tmp1 = vext_u64_u8(simdint::simd_shuffle2(vec, vec, 2, 3), n);
325+
simdint::simd_shuffle4(tmp0, tmp1, 0, 1, 2, 3)
326+
}
320327
}
321328

322329
impl Vector4<u64> for u64x4 {

0 commit comments

Comments
 (0)