Fix performance for CUDA >= 9.2 (GT v1.0-branch) (#1326)

- sets `GT_CONSTEXPR` to `constexpr` for nvcc
- introduces `const_ref` to fix performance for CUDA >= 9.2 (`T` for small data types, `T const&` for large data types)

Backport to GridTools 1.0. See also #1326 and 9bb2d64

Author: lukasm91

include/gridtools/common/array.hpp

@@ -20,6 +20,7 @@
 #include "../meta/macros.hpp"
 #include "../meta/repeat.hpp"
 #include "defs.hpp"
+#include "generic_metafunctions/const_ref.hpp"
 #include "generic_metafunctions/utility.hpp"
 #include "gt_assert.hpp"
 #include "host_device.hpp"
@@ -117,13 +118,13 @@ namespace gridtools {
             }
 
             template <size_t I, typename T, size_t D>
-            static GT_FUNCTION GT_CONSTEXPR const T &get(const array<T, D> &arr) noexcept {
+            static GT_FUNCTION GT_CONSTEXPR GT_META_CALL(const_ref, T) get(const array<T, D> &arr) noexcept {
                 GT_STATIC_ASSERT(I < D, "index is out of bounds");
                 return arr.m_array[I];
             }
 
             template <size_t I, typename T, size_t D>
-            static GT_FUNCTION GT_CONSTEXPR T &&get(array<T, D> &&arr) noexcept {
+            static GT_FUNCTION GT_CONSTEXPR T get(array<T, D> &&arr) noexcept {
                 GT_STATIC_ASSERT(I < D, "index is out of bounds");
                 return wstd::move(arr.m_array[I]);
             }
@@ -188,13 +189,13 @@ namespace gridtools {
     }
 
     template <size_t I, typename T, size_t D>
-    GT_FUNCTION GT_CONSTEXPR const T &get(const array<T, D> &arr) noexcept {
+    GT_FUNCTION GT_CONSTEXPR GT_META_CALL(const_ref, T) get(const array<T, D> &arr) noexcept {
         GT_STATIC_ASSERT(I < D, "index is out of bounds");
         return arr.m_array[I];
     }
 
     template <size_t I, typename T, size_t D>
-    GT_FUNCTION GT_CONSTEXPR T &&get(array<T, D> &&arr) noexcept {
+    GT_FUNCTION GT_CONSTEXPR T get(array<T, D> &&arr) noexcept {
         GT_STATIC_ASSERT(I < D, "index is out of bounds");
         return wstd::move(get<I>(arr));
     }

include/gridtools/common/defs.hpp

@@ -40,11 +40,7 @@
 #endif
 #endif
 
-#ifdef __CUDA_ARCH__
-#define GT_CONSTEXPR
-#else
 #define GT_CONSTEXPR constexpr
-#endif
 
 /**
  * Macro to allow make functions constexpr in c++14 (in case they are not only a return statement)

include/gridtools/common/generic_metafunctions/const_ref.hpp

@@ -0,0 +1,33 @@
+/*
+ * GridTools
+ *
+ * Copyright (c) 2014-2019, ETH Zurich
+ * All rights reserved.
+ *
+ * Please, refer to the LICENSE file in the root directory.
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#pragma once
+
+#include <type_traits>
+
+#include "../../meta/macros.hpp"
+#include "../../meta/type_traits.hpp"
+
+namespace gridtools {
+    GT_META_LAZY_NAMESPACE {
+        template <class T, class = void>
+        struct const_ref : std::add_lvalue_reference<add_const_t<T>> {};
+
+        template <class T>
+        struct const_ref<T,
+            enable_if_t<!std::is_reference<T>::value && std::is_trivially_copy_constructible<T>::value &&
+                        sizeof(T) <= sizeof(add_pointer_t<T>)>> : std::add_const<T> {};
+    }
+
+#if !GT_BROKEN_TEMPLATE_ALIASES
+    template <class T>
+    using const_ref = typename lazy::const_ref<T>::type;
+#endif
+} // namespace gridtools

include/gridtools/common/gt_assert.hpp

@@ -29,6 +29,12 @@
 #endif
 
 #ifdef __CUDA_ARCH__
+#if __CUDACC_VER_MAJOR__ == 9 && __CUDACC_VER_MINOR__ == 2
+// we define this macro to an empty string for CUDA 9.2 because in certain cases, CUDA 9.2 tries to compile device
+// instantiations of certain constexpr function templates, which can lead to compile-time errors like "cannot use an
+// entity undefined in device code".
+#define __PRETTY_FUNCTION__ ""
+#endif
 #define GT_ASSERT_OR_THROW(cond, msg) assert(cond)
 #else
 #define GT_ASSERT_OR_THROW(cond, msg) \

include/gridtools/common/pair.hpp

@@ -13,6 +13,7 @@
 #include <utility>
 
 #include "defs.hpp"
+#include "generic_metafunctions/const_ref.hpp"
 #include "generic_metafunctions/utility.hpp"
 #include "host_device.hpp"
 
@@ -133,30 +134,30 @@ namespace gridtools {
         template <>
         struct pair_get<0> {
             template <typename T1, typename T2>
-            static GT_CONSTEXPR GT_FUNCTION const T1 &const_get(const pair<T1, T2> &p) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION GT_META_CALL(const_ref, T1) const_get(const pair<T1, T2> &p) noexcept {
                 return p.first;
             }
             template <typename T1, typename T2>
             static GT_CONSTEXPR GT_FUNCTION T1 &get(pair<T1, T2> &p) noexcept {
                 return p.first;
             }
             template <typename T1, typename T2>
-            static GT_CONSTEXPR GT_FUNCTION T1 &&move_get(pair<T1, T2> &&p) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION T1 move_get(pair<T1, T2> &&p) noexcept {
                 return wstd::move(p.first);
             }
         };
         template <>
         struct pair_get<1> {
             template <typename T1, typename T2>
-            static GT_CONSTEXPR GT_FUNCTION const T2 &const_get(const pair<T1, T2> &p) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION GT_META_CALL(const_ref, T2) const_get(const pair<T1, T2> &p) noexcept {
                 return p.second;
             }
             template <typename T1, typename T2>
             static GT_CONSTEXPR GT_FUNCTION T2 &get(pair<T1, T2> &p) noexcept {
                 return p.second;
             }
             template <typename T1, typename T2>
-            static GT_CONSTEXPR GT_FUNCTION T2 &&move_get(pair<T1, T2> &&p) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION T2 move_get(pair<T1, T2> &&p) noexcept {
                 return wstd::move(p.second);
             }
         };

include/gridtools/common/tuple.hpp

@@ -15,6 +15,7 @@
 #include "../meta/type_traits.hpp"
 #include "../meta/utility.hpp"
 #include "defs.hpp"
+#include "generic_metafunctions/const_ref.hpp"
 #include "generic_metafunctions/utility.hpp"
 #include "host_device.hpp"
 
@@ -50,7 +51,8 @@ namespace gridtools {
 
         struct tuple_leaf_getter {
             template <size_t I, class T>
-            static GT_CONSTEXPR GT_FUNCTION T const &get(tuple_leaf<I, T, false> const &obj) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION GT_META_CALL(const_ref, T)
+                get(tuple_leaf<I, T, false> const &obj) noexcept {
                 return obj.m_value;
             }
 
@@ -60,12 +62,12 @@ namespace gridtools {
             }
 
             template <size_t I, class T>
-            static GT_CONSTEXPR GT_FUNCTION T &&get(tuple_leaf<I, T, false> &&obj) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION T get(tuple_leaf<I, T, false> &&obj) noexcept {
                 return static_cast<T &&>(get<I>(obj));
             }
 
             template <size_t I, class T>
-            static GT_CONSTEXPR GT_FUNCTION T const &get(tuple_leaf<I, T, true> const &obj) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION GT_META_CALL(const_ref, T) get(tuple_leaf<I, T, true> const &obj) noexcept {
                 return obj;
             }
 
@@ -75,7 +77,7 @@ namespace gridtools {
             }
 
             template <size_t I, class T>
-            static GT_CONSTEXPR GT_FUNCTION T &&get(tuple_leaf<I, T, true> &&obj) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION T get(tuple_leaf<I, T, true> &&obj) noexcept {
                 return static_cast<T &&>(obj);
             }
         };
@@ -168,7 +170,7 @@ namespace gridtools {
         tuple &operator=(tuple const &) = default;
         tuple &operator=(tuple &&) = default;
 
-        GT_CONSTEXPR GT_FUNCTION tuple(Ts const &... args) noexcept : m_impl(args...) {}
+        GT_CONSTEXPR GT_FUNCTION tuple(GT_META_CALL(const_ref, Ts)... args) noexcept : m_impl(args...) {}
 
         template <class... Args,
             enable_if_t<sizeof...(Ts) == sizeof...(Args) && conjunction<std::is_constructible<Ts, Args &&>...>::value,
@@ -198,7 +200,7 @@ namespace gridtools {
         T m_value;
         struct getter {
             template <size_t I, enable_if_t<I == 0, int> = 0>
-            static GT_CONSTEXPR GT_FUNCTION T const &get(tuple const &obj) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION GT_META_CALL(const_ref, T) get(tuple const &obj) noexcept {
                 return obj.m_value;
             }
 
@@ -208,7 +210,7 @@ namespace gridtools {
             }
 
             template <size_t I, enable_if_t<I == 0, int> = 0>
-            static GT_CONSTEXPR GT_FUNCTION T &&get(tuple &&obj) noexcept {
+            static GT_CONSTEXPR GT_FUNCTION T get(tuple &&obj) noexcept {
                 return static_cast<T &&>(obj.m_value);
             }
         };
@@ -225,7 +227,7 @@ namespace gridtools {
         tuple &operator=(tuple const &) = default;
         tuple &operator=(tuple &&) = default;
 
-        GT_CONSTEXPR GT_FUNCTION tuple(T const &arg) noexcept : m_value(arg) {}
+        GT_CONSTEXPR GT_FUNCTION tuple(GT_META_CALL(const_ref, T) arg) noexcept : m_value(arg) {}
 
         template <class Arg, enable_if_t<std::is_constructible<T, Arg &&>::value, int> = 0>
         GT_CONSTEXPR GT_FUNCTION tuple(Arg &&arg) noexcept : m_value(wstd::forward<Arg>(arg)) {}

include/gridtools/common/tuple_util.hpp

@@ -263,10 +263,7 @@ namespace gridtools {
             enum class ref_kind { rvalue, lvalue, const_lvalue };
 
             template <class>
-            struct get_ref_kind;
-
-            template <class T>
-            struct get_ref_kind<T &&> : std::integral_constant<ref_kind, ref_kind::rvalue> {};
+            struct get_ref_kind : std::integral_constant<ref_kind, ref_kind::rvalue> {};
 
             template <class T>
             struct get_ref_kind<T &> : std::integral_constant<ref_kind, ref_kind::lvalue> {};
@@ -279,7 +276,9 @@ namespace gridtools {
                 struct add_ref;
 
                 template <class T>
-                struct add_ref<ref_kind::rvalue, T> : std::add_rvalue_reference<T> {};
+                struct add_ref<ref_kind::rvalue, T> {
+                    using type = T;
+                };
 
                 template <class T>
                 struct add_ref<ref_kind::lvalue, T> : std::add_lvalue_reference<T> {};
@@ -432,7 +431,7 @@ namespace gridtools {
                         class Res = GT_META_CALL(from_types,
                             (Tup,
                                 GT_META_CALL(get_results_t,
-                                    (GT_META_CALL(get_accessors, Tup &&), GT_META_CALL(get_accessors, Tups &&)...))))>
+                                    (GT_META_CALL(get_accessors, Tup), GT_META_CALL(get_accessors, Tups)...))))>
                     GT_TARGET GT_FORCE_INLINE GT_CONSTEXPR Res operator()(Tup &&tup, Tups &&... tups) const {
                         using generators = GT_META_CALL(meta::transform,
                             (get_transform_generator, GT_META_CALL(meta::make_indices_c, size<decay_t<Tup>>::value)));
@@ -506,7 +505,7 @@ namespace gridtools {
 
                     template <class Tup,
                         class Accessors = GT_META_CALL(
-                            meta::transform, (get_accessors, GT_META_CALL(get_accessors, Tup &&))),
+                            meta::transform, (get_accessors, GT_META_CALL(get_accessors, Tup))),
                         class First = GT_META_CALL(meta::first, GT_META_CALL(to_types, Tup)),
                         class Res = GT_META_CALL(from_types, (First, GT_META_CALL(meta::flatten, Accessors)))>
                     GT_TARGET GT_FORCE_INLINE GT_CONSTEXPR Res operator()(Tup &&tup) const {
@@ -524,7 +523,7 @@ namespace gridtools {
                     GT_META_DEFINE_ALIAS(get_drop_front_generator, meta::id, get_nth_f<N + I::value>);
 
                     template <class Tup,
-                        class Accessors = GT_META_CALL(get_accessors, Tup &&),
+                        class Accessors = GT_META_CALL(get_accessors, Tup),
                         class Res = GT_META_CALL(from_types, (Tup, GT_META_CALL(meta::drop_front_c, (N, Accessors))))>
                     GT_TARGET GT_FORCE_INLINE GT_CONSTEXPR Res operator()(Tup &&tup) const {
                         using generators = GT_META_CALL(meta::transform,
@@ -548,7 +547,7 @@ namespace gridtools {
                 struct push_back_f {
                     template <class Tup,
                         class... Args,
-                        class Accessors = GT_META_CALL(get_accessors, Tup &&),
+                        class Accessors = GT_META_CALL(get_accessors, Tup),
                         class Res = GT_META_CALL(
                             from_types, (Tup, GT_META_CALL(meta::push_back, (Accessors, Args &&...))))>
                     GT_TARGET GT_FORCE_INLINE GT_CONSTEXPR Res operator()(Tup &&tup, Args &&... args) const {
@@ -572,7 +571,7 @@ namespace gridtools {
                 struct push_front_f {
                     template <class Tup,
                         class... Args,
-                        class Accessors = GT_META_CALL(get_accessors, Tup &&),
+                        class Accessors = GT_META_CALL(get_accessors, Tup),
                         class Res = GT_META_CALL(
                             from_types, (Tup, GT_META_CALL(meta::push_front, (Accessors, Args &&...))))>
                     GT_TARGET GT_FORCE_INLINE GT_CONSTEXPR Res operator()(Tup &&tup, Args &&... args) const {
@@ -626,7 +625,7 @@ namespace gridtools {
                         size_t N,
                         class State,
                         class Tup,
-                        class AllAccessors = GT_META_CALL(get_accessors, Tup &&),
+                        class AllAccessors = GT_META_CALL(get_accessors, Tup),
                         class Accessors = GT_META_CALL(meta::drop_front_c, (I, AllAccessors)),
                         class Res = GT_META_CALL(meta::lfold, (meta_fun, State &&, Accessors)),
                         enable_if_t<(I + 4 < N), int> = 0>
@@ -643,14 +642,14 @@ namespace gridtools {
 
                     template <class State,
                         class Tup,
-                        class Accessors = GT_META_CALL(get_accessors, Tup &&),
+                        class Accessors = GT_META_CALL(get_accessors, Tup),
                         class Res = GT_META_CALL(meta::lfold, (meta_fun, State &&, Accessors))>
                     GT_TARGET GT_FORCE_INLINE GT_CONSTEXPR Res operator()(State &&state, Tup &&tup) const {
                         return impl<0, size<decay_t<Tup>>::value>(wstd::forward<State>(state), wstd::forward<Tup>(tup));
                     }
 
                     template <class Tup,
-                        class AllAccessors = GT_META_CALL(get_accessors, Tup &&),
+                        class AllAccessors = GT_META_CALL(get_accessors, Tup),
                         class StateAccessor = GT_META_CALL(meta::first, AllAccessors),
                         class Accessors = GT_META_CALL(meta::drop_front_c, (1, AllAccessors)),
                         class Res = GT_META_CALL(meta::lfold, (meta_fun, StateAccessor, Accessors))>
@@ -747,7 +746,7 @@ namespace gridtools {
                     template <class Tup,
                         class First = GT_META_CALL(meta::first, GT_META_CALL(to_types, Tup)),
                         class Accessors = GT_META_CALL(
-                            meta::transform, (get_accessors, GT_META_CALL(get_accessors, Tup &&))),
+                            meta::transform, (get_accessors, GT_META_CALL(get_accessors, Tup))),
                         class Types = GT_META_CALL(meta::transpose, Accessors),
                         class InnerTuples = GT_META_CALL(
                             meta::transform, (get_inner_tuple_f<Tup>::template apply, Types)),
@@ -769,7 +768,7 @@ namespace gridtools {
                     };
 
                     template <class Tup,
-                        class Accessors = GT_META_CALL(get_accessors, Tup &&),
+                        class Accessors = GT_META_CALL(get_accessors, Tup),
                         class Res = GT_META_CALL(from_types, (Tup, GT_META_CALL(meta::reverse, Accessors)))>
                     GT_TARGET GT_FORCE_INLINE GT_CONSTEXPR Res operator()(Tup &&tup) const {
                         using n_t = size<decay_t<Tup>>;
@@ -809,7 +808,7 @@ namespace gridtools {
                                 (I::value == N, insert_val_generator_f, insert_tup_generator_f<I::value - 1>))));
 
                     template <class Tup,
-                        class Accessors = GT_META_CALL(get_accessors, Tup &&),
+                        class Accessors = GT_META_CALL(get_accessors, Tup),
                         class Types = GT_META_CALL(meta::insert_c, (N, Accessors, Val)),
                         class Res = GT_META_CALL(from_types, (Tup, Types))>
                     GT_TARGET GT_FORCE_INLINE GT_CONSTEXPR Res operator()(Tup &&tup) const {

include/gridtools/stencil_composition/expressions/expr_base.hpp

@@ -63,8 +63,15 @@ namespace gridtools {
                 return arg;
             }
 
+            // intel compiler 18.0 segfaults if this is a value. On the other hand, nvcc performs much worse in the
+            // dycore if it is a lvalue reference
+#if defined(__INTEL_COMPILER) && (__INTEL_COMPILER <= 1800)
             template <class Eval, class Arg, enable_if_t<!std::is_arithmetic<Arg>::value, int> = 0>
             GT_FUNCTION GT_CONSTEXPR auto apply_eval(Eval &eval, Arg const &arg) GT_AUTO_RETURN(eval(arg));
+#else
+            template <class Eval, class Arg, enable_if_t<!std::is_arithmetic<Arg>::value, int> = 0>
+            GT_FUNCTION GT_CONSTEXPR auto apply_eval(Eval &eval, Arg arg) GT_AUTO_RETURN(eval(wstd::move(arg)));
+#endif
 
             template <class Eval, class Op, class Arg>
             GT_FUNCTION GT_CONSTEXPR auto value(Eval &eval, expr<Op, Arg> const &arg)

include/gridtools/stencil_composition/sid/composite.hpp

@@ -95,7 +95,8 @@ namespace gridtools {
                     using type = item_generator;
 
                     template <class Args, class Res = GT_META_CALL(tuple_util::element, (PrimaryIndex::value, Args))>
-                    Res const &operator()(Args const &args) const noexcept {
+                    GT_META_CALL(const_ref, Res)
+                    operator()(Args const &args) const noexcept {
                         GT_STATIC_ASSERT(
                             (conjunction<
                                 std::is_same<GT_META_CALL(tuple_util::element, (SecondaryIndices::value, Args)),

include/gridtools/stencil_composition/stencil_functions.hpp

@@ -135,9 +135,8 @@ namespace gridtools {
             Transforms m_transforms;
 
             template <class Accessor>
-            GT_FUNCTION auto operator()(Accessor &&acc) const
-                GT_AUTO_RETURN(tuple_util::host_device::get<decay_t<Accessor>::index_t::value>(m_transforms)(
-                    m_eval, std::forward<Accessor>(acc)));
+            GT_FUNCTION auto operator()(Accessor acc) const GT_AUTO_RETURN(
+                tuple_util::host_device::get<decay_t<Accessor>::index_t::value>(m_transforms)(m_eval, wstd::move(acc)));
 
             template <class Op, class... Ts>
             GT_FUNCTION auto operator()(expr<Op, Ts...> const &arg) const
@@ -227,12 +226,12 @@ namespace gridtools {
          */
         template <class Eval,
             class... Args,
-            class Res = typename call_interfaces_impl_::get_result_type<Eval, ReturnType, decay_t<Args>...>::type,
+            class Res = typename call_interfaces_impl_::get_result_type<Eval, ReturnType, Args...>::type,
             enable_if_t<sizeof...(Args) + 1 == meta::length<params_t>::value, int> = 0>
-        GT_FUNCTION static Res with(Eval &eval, Args &&... args) {
+        GT_FUNCTION static Res with(Eval &eval, Args... args) {
             Res res;
-            call_interfaces_impl_::evaluate_bound_functor<Functor, Region, OffI, OffJ, OffK>(eval,
-                tuple_util::host_device::insert<out_param_index>(res, tuple<Args &&...>{std::forward<Args>(args)...}));
+            call_interfaces_impl_::evaluate_bound_functor<Functor, Region, OffI, OffJ, OffK>(
+                eval, tuple_util::host_device::insert<out_param_index>(res, tuple<Args &&...>{wstd::move(args)...}));
             return res;
         }
     };