Support s390x z13 vector ABI

Author: taiki-e

compiler/rustc_codegen_gcc/src/builder.rs

@@ -30,7 +30,7 @@ use rustc_middle::ty::{Instance, ParamEnv, Ty, TyCtxt};
 use rustc_span::Span;
 use rustc_span::def_id::DefId;
 use rustc_target::abi::call::FnAbi;
-use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, WasmCAbi};
+use rustc_target::spec::{HasS390xVector, HasTargetSpec, HasWasmCAbiOpt, Target, WasmCAbi};
 
 use crate::common::{SignType, TypeReflection, type_is_pointer};
 use crate::context::CodegenCx;
@@ -2347,6 +2347,12 @@ impl<'tcx> HasWasmCAbiOpt for Builder<'_, '_, 'tcx> {
     }
 }
 
+impl<'tcx> HasS390xVector for Builder<'_, '_, 'tcx> {
+    fn has_s390x_vector(&self) -> bool {
+        self.cx.has_s390x_vector()
+    }
+}
+
 pub trait ToGccComp {
     fn to_gcc_comparison(&self) -> ComparisonOp;
 }

compiler/rustc_codegen_gcc/src/context.rs

@@ -19,7 +19,9 @@ use rustc_session::Session;
 use rustc_span::source_map::respan;
 use rustc_span::{DUMMY_SP, Span};
 use rustc_target::abi::{HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx};
-use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, TlsModel, WasmCAbi};
+use rustc_target::spec::{
+    HasS390xVector, HasTargetSpec, HasWasmCAbiOpt, Target, TlsModel, WasmCAbi,
+};
 
 use crate::callee::get_fn;
 use crate::common::SignType;
@@ -538,6 +540,13 @@ impl<'gcc, 'tcx> HasWasmCAbiOpt for CodegenCx<'gcc, 'tcx> {
     }
 }
 
+impl<'gcc, 'tcx> HasS390xVector for CodegenCx<'gcc, 'tcx> {
+    fn has_s390x_vector(&self) -> bool {
+        // `unstable_target_features` is used here because "vector" is gated behind s390x_target_feature.
+        self.tcx.sess.unstable_target_features.contains(&sym::vector)
+    }
+}
+
 impl<'gcc, 'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'gcc, 'tcx> {
     #[inline]
     fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {

compiler/rustc_middle/src/ty/layout.rs

@@ -21,7 +21,9 @@ use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
 use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::{FieldIdx, TyAbiInterface, VariantIdx, call};
 use rustc_target::spec::abi::Abi as SpecAbi;
-use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, PanicStrategy, Target, WasmCAbi};
+use rustc_target::spec::{
+    HasS390xVector, HasTargetSpec, HasWasmCAbiOpt, PanicStrategy, Target, WasmCAbi,
+};
 use tracing::debug;
 use {rustc_abi as abi, rustc_hir as hir};
 
@@ -544,6 +546,13 @@ impl<'tcx> HasWasmCAbiOpt for TyCtxt<'tcx> {
     }
 }
 
+impl<'tcx> HasS390xVector for TyCtxt<'tcx> {
+    fn has_s390x_vector(&self) -> bool {
+        // `unstable_target_features` is used here because "vector" is gated behind s390x_target_feature.
+        self.sess.unstable_target_features.contains(&sym::vector)
+    }
+}
+
 impl<'tcx> HasTyCtxt<'tcx> for TyCtxt<'tcx> {
     #[inline]
     fn tcx(&self) -> TyCtxt<'tcx> {
@@ -595,6 +604,12 @@ impl<'tcx> HasWasmCAbiOpt for LayoutCx<'tcx> {
     }
 }
 
+impl<'tcx> HasS390xVector for LayoutCx<'tcx> {
+    fn has_s390x_vector(&self) -> bool {
+        self.calc.cx.has_s390x_vector()
+    }
+}
+
 impl<'tcx> HasTyCtxt<'tcx> for LayoutCx<'tcx> {
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.calc.cx

compiler/rustc_span/src/symbol.rs

@@ -2126,6 +2126,7 @@ symbols! {
         vec_pop,
         vec_with_capacity,
         vecdeque_iter,
+        vector,
         version,
         vfp2,
         vis,

compiler/rustc_target/src/abi/call/mod.rs

@@ -5,7 +5,7 @@ use rustc_macros::HashStable_Generic;
 use rustc_span::Symbol;
 
 use crate::abi::{self, Abi, Align, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout};
-use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt, WasmCAbi};
+use crate::spec::{self, HasS390xVector, HasTargetSpec, HasWasmCAbiOpt, WasmCAbi};
 
 mod aarch64;
 mod amdgpu;
@@ -876,7 +876,7 @@ impl<'a, Ty> FnAbi<'a, Ty> {
     ) -> Result<(), AdjustForForeignAbiError>
     where
         Ty: TyAbiInterface<'a, C> + Copy,
-        C: HasDataLayout + HasTargetSpec + HasWasmCAbiOpt,
+        C: HasDataLayout + HasTargetSpec + HasWasmCAbiOpt + HasS390xVector,
     {
         if abi == spec::abi::Abi::X86Interrupt {
             if let Some(arg) = self.args.first_mut() {

compiler/rustc_target/src/abi/call/s390x.rs

@@ -1,19 +1,49 @@
-// FIXME: The assumes we're using the non-vector ABI, i.e., compiling
-// for a pre-z13 machine or using -mno-vx.
+use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind};
+use crate::abi::{Abi, HasDataLayout, TyAbiInterface, TyAndLayout};
+use crate::spec::{HasS390xVector, HasTargetSpec};
 
-use crate::abi::call::{ArgAbi, FnAbi, Reg};
-use crate::abi::{HasDataLayout, TyAbiInterface};
-use crate::spec::HasTargetSpec;
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum ABI {
+    NoVector, // no-vector ABI, i.e., compiling for a pre-z13 machine or using -C target-feature=-vector
+    Vector, // vector ABI, i.e., compiling for a z13 or later machine or using -C target-feature=+vector
+}
+use ABI::*;
+
+fn contains_vector<'a, Ty, C>(cx: &C, layout: TyAndLayout<'a, Ty>) -> bool
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+{
+    match layout.abi {
+        Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) => false,
+        Abi::Vector { .. } => layout.size.bits() == 128,
+        Abi::Aggregate { .. } => {
+            for i in 0..layout.fields.count() {
+                if contains_vector(cx, layout.field(cx, i)) {
+                    return true;
+                }
+            }
+            false
+        }
+    }
+}
 
-fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
-    if !ret.layout.is_aggregate() && ret.layout.size.bits() <= 64 {
+fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>, abi: ABI)
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+{
+    let size = ret.layout.size;
+    if !ret.layout.is_aggregate() && size.bits() <= 64 {
         ret.extend_integer_width_to(64);
-    } else {
-        ret.make_indirect();
+        return;
+    }
+    if abi == Vector && size.bits() == 128 && contains_vector(cx, ret.layout) {
+        ret.cast_to(Reg { kind: RegKind::Vector, size });
+        return;
     }
+    ret.make_indirect();
 }
 
-fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
+fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, abi: ABI)
 where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout + HasTargetSpec,
@@ -32,19 +62,25 @@ where
         }
         return;
     }
-    if !arg.layout.is_aggregate() && arg.layout.size.bits() <= 64 {
+
+    let size = arg.layout.size;
+    if !arg.layout.is_aggregate() && size.bits() <= 64 {
         arg.extend_integer_width_to(64);
         return;
     }
+    if abi == Vector && size.bits() == 128 && contains_vector(cx, arg.layout) {
+        arg.cast_to(Reg { kind: RegKind::Vector, size });
+        return;
+    }
 
     if arg.layout.is_single_fp_element(cx) {
-        match arg.layout.size.bytes() {
+        match size.bytes() {
             4 => arg.cast_to(Reg::f32()),
             8 => arg.cast_to(Reg::f64()),
             _ => arg.make_indirect(),
         }
     } else {
-        match arg.layout.size.bytes() {
+        match size.bytes() {
             1 => arg.cast_to(Reg::i8()),
             2 => arg.cast_to(Reg::i16()),
             4 => arg.cast_to(Reg::i32()),
@@ -57,13 +93,15 @@ where
 pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
 where
     Ty: TyAbiInterface<'a, C> + Copy,
-    C: HasDataLayout + HasTargetSpec,
+    C: HasDataLayout + HasTargetSpec + HasS390xVector,
 {
+    let abi = if cx.has_s390x_vector() { Vector } else { NoVector };
+
     if !fn_abi.ret.is_ignore() {
-        classify_ret(&mut fn_abi.ret);
+        classify_ret(cx, &mut fn_abi.ret, abi);
     }
 
     for arg in fn_abi.args.iter_mut() {
-        classify_arg(cx, arg);
+        classify_arg(cx, arg, abi);
     }
 }

compiler/rustc_target/src/spec/mod.rs

@@ -2096,6 +2096,10 @@ pub trait HasWasmCAbiOpt {
     fn wasm_c_abi_opt(&self) -> WasmCAbi;
 }
 
+pub trait HasS390xVector {
+    fn has_s390x_vector(&self) -> bool;
+}
+
 type StaticCow<T> = Cow<'static, T>;
 
 /// Optional aspects of a target specification.

compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs

@@ -6,9 +6,6 @@ pub(crate) fn target() -> Target {
     base.endian = Endian::Big;
     // z10 is the oldest CPU supported by LLVM
     base.cpu = "z10".into();
-    // FIXME: The ABI implementation in abi/call/s390x.rs is for now hard-coded to assume the no-vector
-    // ABI. Pass the -vector feature string to LLVM to respect this assumption.
-    base.features = "-vector".into();
     base.max_atomic_width = Some(128);
     base.min_global_align = Some(16);
     base.stack_probes = StackProbeType::Inline;

compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs

@@ -6,9 +6,6 @@ pub(crate) fn target() -> Target {
     base.endian = Endian::Big;
     // z10 is the oldest CPU supported by LLVM
     base.cpu = "z10".into();
-    // FIXME: The ABI implementation in abi/call/s390x.rs is for now hard-coded to assume the no-vector
-    // ABI. Pass the -vector feature string to LLVM to respect this assumption.
-    base.features = "-vector".into();
     base.max_atomic_width = Some(128);
     base.min_global_align = Some(16);
     base.static_position_independent_executables = true;

tests/assembly/s390x-vector-abi.rs

@@ -0,0 +1,113 @@
+//@ revisions: z10 z10_vector z13 z13_no_vector
+//@ assembly-output: emit-asm
+//@ compile-flags: -O
+//@[z10] compile-flags: --target s390x-unknown-linux-gnu --cfg no_vector
+//@[z10] needs-llvm-components: systemz
+//@[z10_vector] compile-flags: --target s390x-unknown-linux-gnu --cfg vector
+//@[z10_vector] needs-llvm-components: systemz
+//@[z13] compile-flags: --target s390x-unknown-linux-gnu -C target-cpu=z13 --cfg vector
+//@[z13] needs-llvm-components: systemz
+//@[z13_no_vector] compile-flags: --target s390x-unknown-linux-gnu -C target-cpu=z13 -C target-feature=-vector --cfg no_vector
+//@[z13_no_vector] needs-llvm-components: systemz
+
+#![feature(no_core, lang_items, repr_simd)]
+#![no_core]
+#![crate_type = "lib"]
+#![allow(non_camel_case_types)]
+
+#[lang = "sized"]
+trait Sized {}
+#[lang = "copy"]
+trait Copy {}
+#[lang = "freeze"]
+trait Freeze {}
+
+impl<T: Copy, const N: usize> Copy for [T; N] {}
+
+#[repr(simd)]
+pub struct i8x8([i8; 8]);
+#[repr(simd)]
+pub struct i8x16([i8; 16]);
+#[repr(simd)]
+pub struct i8x32([i8; 32]);
+pub struct Wrapper<T>(T);
+
+impl Copy for i8 {}
+impl Copy for i8x8 {}
+impl Copy for i8x16 {}
+impl Copy for i8x32 {}
+impl<T: Copy> Copy for Wrapper<T> {}
+
+// FIXME: ??
+// CHECK-LABEL: vector_small
+// no_vector: lg %r0, 0(%r3)
+// no_vector-NEXT: stg %r0, 0(%r2)
+// no_vector-NEXT: br %r14
+// vector: vlrepg %v24, 0(%r2)
+// vector-NEXT: br %r14
+#[no_mangle]
+extern "C" fn vector_small(x: &i8x8) -> i8x8 {
+    *x
+}
+// CHECK-LABEL: vector
+// no_vector: mvc 8(8,%r2), 8(%r3)
+// no_vector-NEXT: mvc 0(8,%r2), 0(%r3)
+// no_vector-NEXT: br %r14
+// vector: vl %v24, 0(%r2), 3
+// vector-NEXT: br %r14
+#[no_mangle]
+extern "C" fn vector(x: &i8x16) -> i8x16 {
+    *x
+}
+// CHECK-LABEL: vector_large
+// no_vector: mvc 24(8,%r2), 24(%r3)
+// no_vector-NEXT: mvc 16(8,%r2), 16(%r3)
+// no_vector-NEXT: mvc 8(8,%r2), 8(%r3)
+// no_vector-NEXT: mvc 0(8,%r2), 0(%r3)
+// no_vector-NEXT: br %r14
+// vector: vl %v0, 0(%r3), 4
+// vector-NEXT: vl %v1, 16(%r3), 4
+// vector-NEXT: vst %v1, 16(%r2), 4
+// vector-NEXT: vst %v0, 0(%r2), 4
+// vector-NEXT: br %r14
+#[no_mangle]
+extern "C" fn vector_large(x: &i8x32) -> i8x32 {
+    *x
+}
+
+// FIXME: ??
+// CHECK-LABEL: vector_wrapper_small
+// no_vector: lg %r0, 0(%r3)
+// no_vector-NEXT: stg %r0, 0(%r2)
+// no_vector-NEXT: br %r14
+// vector: vlrepg %v24, 0(%r2)
+// vector-NEXT: br %r14
+#[no_mangle]
+extern "C" fn vector_wrapper_small(x: &Wrapper<i8x8>) -> Wrapper<i8x8> {
+    *x
+}
+// CHECK-LABEL: vector_wrapper
+// no_vector: mvc 8(8,%r2), 8(%r3)
+// no_vector-NEXT: mvc 0(8,%r2), 0(%r3)
+// no_vector-NEXT: br %r14
+// vector: vl %v24, 0(%r2), 3
+// vector-NEXT: br %r14
+#[no_mangle]
+extern "C" fn vector_wrapper(x: &Wrapper<i8x16>) -> Wrapper<i8x16> {
+    *x
+}
+// CHECK-LABEL: vector_wrapper_large
+// no_vector: mvc 24(8,%r2), 24(%r3)
+// no_vector-NEXT: mvc 16(8,%r2), 16(%r3)
+// no_vector-NEXT: mvc 8(8,%r2), 8(%r3)
+// no_vector-NEXT: mvc 0(8,%r2), 0(%r3)
+// no_vector-NEXT: br %r14
+// vector: vl %v0, 0(%r3), 4
+// vector-NEXT: vl %v1, 16(%r3), 4
+// vector-NEXT: vst %v1, 16(%r2), 4
+// vector-NEXT: vst %v0, 0(%r2), 4
+// vector-NEXT: br %r14
+#[no_mangle]
+extern "C" fn vector_wrapper_large(x: &Wrapper<i8x32>) -> Wrapper<i8x32> {
+    *x
+}