Skip to content

Don't set fast-math for the SIMD operations we set it for previously #84274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 18, 2021
Merged

Don't set fast-math for the SIMD operations we set it for previously #84274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions compiler/rustc_codegen_llvm/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,39 +261,39 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down Expand Up @@ -1242,14 +1242,14 @@ impl Builder<'a, 'll, 'tcx> {
pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down Expand Up @@ -1282,15 +1282,15 @@ impl Builder<'a, 'll, 'tcx> {
unsafe {
let instr =
llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value {
unsafe {
let instr =
llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down
40 changes: 19 additions & 21 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1053,50 +1053,48 @@ fn generic_simd_intrinsic(
let vec_ty = bx.type_vector(elem_ty, in_len);

let (intr_name, fn_ty) = match name {
sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
_ => return_error!("unrecognized intrinsic `{}`", name),
};

let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty);
let c = bx.call(f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) };
Ok(c)
}

if std::matches!(
name,
sym::simd_fsqrt
| sym::simd_fsin
| sym::simd_fcos
sym::simd_ceil
| sym::simd_fabs
| sym::simd_ceil
| sym::simd_floor
| sym::simd_round
| sym::simd_trunc
| sym::simd_fexp
| sym::simd_fcos
| sym::simd_fexp2
| sym::simd_fexp
| sym::simd_flog10
| sym::simd_flog2
| sym::simd_flog
| sym::simd_fpowi
| sym::simd_fpow
| sym::simd_floor
| sym::simd_fma
| sym::simd_fpow
| sym::simd_fpowi
| sym::simd_fsin
| sym::simd_fsqrt
| sym::simd_round
| sym::simd_trunc
) {
return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
}
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1354,7 +1354,7 @@ extern "C" {
pub fn LLVMBuildNeg(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMBuildFNeg(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMBuildNot(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMRustSetHasUnsafeAlgebra(Instr: &Value);
pub fn LLVMRustSetFastMath(Instr: &Value);

// Memory
pub fn LLVMBuildAlloca(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value;
Expand Down
6 changes: 4 additions & 2 deletions compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,8 +349,10 @@ extern "C" void LLVMRustRemoveFunctionAttributes(LLVMValueRef Fn,
F->setAttributes(PALNew);
}

// enable fpmath flag UnsafeAlgebra
extern "C" void LLVMRustSetHasUnsafeAlgebra(LLVMValueRef V) {
// Enable a fast-math flag
//
// https://llvm.org/docs/LangRef.html#fast-math-flags
extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) {
I->setFast(true);
}
Expand Down
23 changes: 23 additions & 0 deletions src/test/codegen/issue-84268.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// compile-flags: -O --crate-type=rlib
#![feature(platform_intrinsics, repr_simd)]

extern "platform-intrinsic" {
fn simd_fabs<T>(x: T) -> T;
fn simd_eq<T, U>(x: T, y: T) -> U;
}

#[repr(simd)]
pub struct V([f32; 4]);

#[repr(simd)]
pub struct M([i32; 4]);

#[no_mangle]
// CHECK-LABEL: @is_infinite
pub fn is_infinite(v: V) -> M {
// CHECK: fabs
// CHECK: cmp oeq
unsafe {
simd_eq(simd_fabs(v), V([f32::INFINITY; 4]))
}
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-abs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @fabs_32x2
#[no_mangle]
pub unsafe fn fabs_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.fabs.v2f32
// CHECK: call <2 x float> @llvm.fabs.v2f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x4
#[no_mangle]
pub unsafe fn fabs_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.fabs.v4f32
// CHECK: call <4 x float> @llvm.fabs.v4f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x8
#[no_mangle]
pub unsafe fn fabs_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.fabs.v8f32
// CHECK: call <8 x float> @llvm.fabs.v8f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x16
#[no_mangle]
pub unsafe fn fabs_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.fabs.v16f32
// CHECK: call <16 x float> @llvm.fabs.v16f32
simd_fabs(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @fabs_64x4
#[no_mangle]
pub unsafe fn fabs_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.fabs.v4f64
// CHECK: call <4 x double> @llvm.fabs.v4f64
simd_fabs(a)
}

// CHECK-LABEL: @fabs_64x2
#[no_mangle]
pub unsafe fn fabs_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.fabs.v2f64
// CHECK: call <2 x double> @llvm.fabs.v2f64
simd_fabs(a)
}

// CHECK-LABEL: @fabs_64x8
#[no_mangle]
pub unsafe fn fabs_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.fabs.v8f64
// CHECK: call <8 x double> @llvm.fabs.v8f64
simd_fabs(a)
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-ceil.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @ceil_32x2
#[no_mangle]
pub unsafe fn ceil_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.ceil.v2f32
// CHECK: call <2 x float> @llvm.ceil.v2f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x4
#[no_mangle]
pub unsafe fn ceil_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.ceil.v4f32
// CHECK: call <4 x float> @llvm.ceil.v4f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x8
#[no_mangle]
pub unsafe fn ceil_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.ceil.v8f32
// CHECK: call <8 x float> @llvm.ceil.v8f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x16
#[no_mangle]
pub unsafe fn ceil_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.ceil.v16f32
// CHECK: call <16 x float> @llvm.ceil.v16f32
simd_ceil(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @ceil_64x4
#[no_mangle]
pub unsafe fn ceil_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.ceil.v4f64
// CHECK: call <4 x double> @llvm.ceil.v4f64
simd_ceil(a)
}

// CHECK-LABEL: @ceil_64x2
#[no_mangle]
pub unsafe fn ceil_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.ceil.v2f64
// CHECK: call <2 x double> @llvm.ceil.v2f64
simd_ceil(a)
}

// CHECK-LABEL: @ceil_64x8
#[no_mangle]
pub unsafe fn ceil_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.ceil.v8f64
// CHECK: call <8 x double> @llvm.ceil.v8f64
simd_ceil(a)
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-cos.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @fcos_32x2
#[no_mangle]
pub unsafe fn fcos_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.cos.v2f32
// CHECK: call <2 x float> @llvm.cos.v2f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x4
#[no_mangle]
pub unsafe fn fcos_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.cos.v4f32
// CHECK: call <4 x float> @llvm.cos.v4f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x8
#[no_mangle]
pub unsafe fn fcos_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.cos.v8f32
// CHECK: call <8 x float> @llvm.cos.v8f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x16
#[no_mangle]
pub unsafe fn fcos_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.cos.v16f32
// CHECK: call <16 x float> @llvm.cos.v16f32
simd_fcos(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @fcos_64x4
#[no_mangle]
pub unsafe fn fcos_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.cos.v4f64
// CHECK: call <4 x double> @llvm.cos.v4f64
simd_fcos(a)
}

// CHECK-LABEL: @fcos_64x2
#[no_mangle]
pub unsafe fn fcos_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.cos.v2f64
// CHECK: call <2 x double> @llvm.cos.v2f64
simd_fcos(a)
}

// CHECK-LABEL: @fcos_64x8
#[no_mangle]
pub unsafe fn fcos_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.cos.v8f64
// CHECK: call <8 x double> @llvm.cos.v8f64
simd_fcos(a)
}
Loading