lammps for todi gh200 (#54)

config.yaml

@@ -107,6 +107,13 @@ uenvs:
       deploy:
         eiger: [zen2]
       develop: False
+    "2024":
+      recipes:
+        gh200: "2024/gh200"
+      deploy:
+        todi: [gh200]
+        santis: [gh200]
+      develop: False
   linaro-forge:
     "23.1.2":
       recipes:

docs/uenv-lammps.md

@@ -0,0 +1,36 @@
+# LAMMPS
+
+[LAMMPS] is a classical molecular dynamics code with a focus on materials modeling. It's an acronym for Large-scale Atomic/Molecular Massively Parallel Simulator. LAMMPS has potentials for solid-state materials (metals, semiconductors) and soft matter (biomolecules, polymers) and coarse-grained or mesoscopic systems. It can be used to model atoms or, more generically, as a parallel particle simulator at the atomic, meso, or continuum scale. See [LAMMPS Features] for a detailed overview.
+
+## Running
+
+```bash
+uenv start <LAMMPS_UENV>
+uenv view lammps
+```
+
+!!! warning
+    [LAMMPS] is built with GPU-aware MPI. Make sure to set `MPICH_GPU_SUPPORT_ENABLED=1` when running [LAMMPS].
+
+## Building from source
+
+The [LAMMPS] `uenv` provides all the dependencies required to build [LAMMPS] from source, including kokkos. You can follow these steps to build [LAMMPS] from source:
+
+```bash
+# Start uenv and load develop view
+uenv start <LAMMPS_UENV>
+uenv view develop
+
+# cd to LAMMPS source directory
+cd <PATH_TO_LAMMPS_SOURCE>
+
+# CMake
+mkdir build && cd build
+cmake -C ../cmake/presets/kokkos-cuda.cmake ../cmake/  -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF -DKokkos_ARCH_NATIVE=yes -DKokkos_ARCH_HOPPER90=yes
+
+cmake --build . --parallel 32
+```
+
+[LAMMPS]: https://www.lammps.org/
+[LAMMPS Features]: https://docs.lammps.org/Intro_features.html
+

recipes/lammps/2024/gh200/compilers.yaml

@@ -0,0 +1,5 @@
+bootstrap:
+  spec: gcc@12
+gcc:
+  specs:
+  - [email protected]

recipes/lammps/2024/gh200/config.yaml

@@ -0,0 +1,7 @@
+name: lammps
+store: /user-environment
+description: lammps build targetting GH200
+spack:
+  repo: https://github.com/spack/spack.git
+  commit: v0.22.0
+modules: true

recipes/lammps/2024/gh200/environments.yaml

@@ -0,0 +1,26 @@
+lammps:
+  compiler:
+  - toolchain: gcc
+    spec: [email protected]
+  mpi:
+    spec: cray-mpich
+    gpu: cuda
+  unify: true
+  specs:
+  - [email protected]
+  - cmake
+  - cuda@12
+  - [email protected] +wrapper +cuda cuda_arch=90
+  - mpi
+  - cray-mpich
+  - [email protected] +cuda cuda_arch=90 +python +kokkos
+  variants:
+  - +mpi
+  - +cuda
+  - cuda_arch=90
+  views:
+    develop:
+      link: roots
+      exclude: ["lammps"]
+    lammps:
+      link: roots

recipes/lammps/2024/gh200/modules.yaml

@@ -0,0 +1,23 @@
+modules:
+  # Paths to check when creating modules for all module sets
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    # Where to install modules
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: ['%[email protected]', 'gcc %[email protected]']
+      projections:
+        all: '{name}/{version}'

recipes/lammps/2024/gh200/repo/packages/kokkos/hpx_profiling_fences.patch

@@ -0,0 +1,84 @@
+diff --git a/core/src/HPX/Kokkos_HPX_Task.hpp b/core/src/HPX/Kokkos_HPX_Task.hpp
+index 7bb3ca5d0..ff50fdc5f 100644
+--- a/core/src/HPX/Kokkos_HPX_Task.hpp
++++ b/core/src/HPX/Kokkos_HPX_Task.hpp
+@@ -216,7 +216,7 @@ class TaskQueueSpecializationConstrained<
+     task_queue.scheduler = &scheduler;
+     Kokkos::Impl::dispatch_execute_task(&task_queue,
+                                         Kokkos::Experimental::HPX());
+-    Kokkos::Experimental::HPX().fence()"Kokkos::Impl::TaskQueueSpecializationConstrained::execute: fence after task execution";
++    Kokkos::Experimental::HPX().fence("Kokkos::Impl::TaskQueueSpecializationConstrained::execute: fence after task execution");
+   }
+
+   // Must provide task queue execution function
+diff --git a/core/src/Kokkos_HPX.hpp b/core/src/Kokkos_HPX.hpp
+index 236211864..3e8522e94 100644
+--- a/core/src/Kokkos_HPX.hpp
++++ b/core/src/Kokkos_HPX.hpp
+@@ -282,11 +282,11 @@ class HPX {
+     m_mode                      = other.m_mode;
+     m_independent_instance_data = other.m_independent_instance_data;
+     m_buffer                    = m_mode == instance_mode::independent
+-                   ? m_independent_instance_data->m_buffer
+-                   : m_global_instance_data.m_buffer;
+-    m_future = m_mode == instance_mode::independent
+-                   ? m_independent_instance_data->m_future
+-                   : m_global_instance_data.m_future;
++                                      ? m_independent_instance_data->m_buffer
++                                      : m_global_instance_data.m_buffer;
++    m_future                    = m_mode == instance_mode::independent
++                                      ? m_independent_instance_data->m_future
++                                      : m_global_instance_data.m_future;
+     return *this;
+   }
+ #else
+@@ -322,25 +322,36 @@ class HPX {
+         "Fence");
+   }
+   void impl_fence_instance(const std::string &name) const {
+-    Kokkos::Tools::Experimental::Impl::profile_fence_event(name, *this, [&]() {
+-      if (hpx::threads::get_self_ptr() == nullptr) {
+-        hpx::threads::run_as_hpx_thread([this]() { impl_get_future().wait(); });
+-      } else {
+-        impl_get_future().wait();
+-      }
+-    });
++    Kokkos::Tools::Experimental::Impl::profile_fence_event<
++        Kokkos::Experimental::HPX>(
++        name,
++        Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{
++            impl_instance_id()},
++        [&]() {
++          if (hpx::threads::get_self_ptr() == nullptr) {
++            hpx::threads::run_as_hpx_thread(
++                [this]() { impl_get_future().wait(); });
++          } else {
++            impl_get_future().wait();
++          }
++        });
+   }
+
+   void impl_fence_all_instances() const {
+-    impl_fence_instance(
++    impl_fence_all_instances(
+         "Kokkos::Experimental::HPX::impl_fence_all_instances: Unnamed Global "
+         "HPX Fence");
+   }
+-  void impl_fence_all_instances(const std::string &namename) const {
+-    Kokkos::Tools::Experimental::Impl::profile_fence_event(name, *this, [&]() {
+-      hpx::util::yield_while(
+-          []() { return m_active_parallel_region_count.load() != 0; });
+-    });
++  void impl_fence_all_instances(const std::string &name) const {
++    Kokkos::Tools::Experimental::Impl::profile_fence_event<
++        Kokkos::Experimental::HPX>(
++        name,
++        Kokkos::Tools::Experimental::SpecialSynchronizationCases::
++            GlobalDeviceSynchronization,
++        [&]() {
++          hpx::util::yield_while(
++              []() { return m_active_parallel_region_count.load() != 0; });
++        });
+   }
+ #endif
+