From 8ff5ac5b7a25ca1384524448867b9886f4ce06b3 Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Wed, 12 Feb 2025 13:09:56 +0100 Subject: [PATCH 01/17] Add openmpi@5 using libfabric and cxi build from source --- stackinator/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 66293147..de1aa8f0 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -15,7 +15,7 @@ class Recipe: "3.0a", "+xpmem fabrics=ch4ofi ch4_max_vcis=4 process_managers=slurm", ), - "openmpi": ("5", "+internal-pmix +legacylaunchers +orterunprefix fabrics=cma,ofi,xpmem schedulers=slurm"), + "openmpi": ("5", "+internal-pmix fabrics=cma,ofi,xpmem schedulers=slurm +cray-xpmem ^libfabric@main ^libcxi@main ^cxi-driver@main ^cassini-headers@main"), } @property From e363de94f1cf94d147c7a54dfb5afdc07421332a Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Wed, 12 Feb 2025 15:15:25 +0100 Subject: [PATCH 02/17] Do not add '+cuda' after a spec containing an explicit ^dependency a spec such as openmpi ^cassini-headers@main +cuda is invalid since the+cuda variant should be applied to the openmpi spec and not the cassini-headers one. This patch makes sure that extra variants are added before dependents, eg openmpi +cuda ^cassini-headers@main --- stackinator/recipe.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index de1aa8f0..eb5256d2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -311,7 +311,10 @@ def generate_environment_specs(self, raw): spec = f"{mpi_impl}{version_opt} {options or ''}".strip() if mpi_gpu: - spec = f"{spec} +{mpi_gpu}" + if '^' in spec: + spec = spec.replace('^', f'+{mpi_gpu} ^', 1) + else: + spec = f"{spec} +{mpi_gpu}" environments[name]["specs"].append(spec) else: From 4d8d0dbcb7cb11f9c46f1c3faa8705f25fa8d6fc Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Wed, 12 Feb 2025 16:36:24 +0100 Subject: [PATCH 03/17] reformat using black --- stackinator/recipe.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index eb5256d2..68d91887 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -15,7 +15,10 @@ class Recipe: "3.0a", "+xpmem fabrics=ch4ofi ch4_max_vcis=4 process_managers=slurm", ), - "openmpi": ("5", "+internal-pmix fabrics=cma,ofi,xpmem schedulers=slurm +cray-xpmem ^libfabric@main ^libcxi@main ^cxi-driver@main ^cassini-headers@main"), + "openmpi": ( + "5", + "+internal-pmix fabrics=cma,ofi,xpmem schedulers=slurm +cray-xpmem ^libfabric@main ^libcxi@main ^cxi-driver@main ^cassini-headers@main", + ), } @property @@ -311,8 +314,8 @@ def generate_environment_specs(self, raw): spec = f"{mpi_impl}{version_opt} {options or ''}".strip() if mpi_gpu: - if '^' in spec: - spec = spec.replace('^', f'+{mpi_gpu} ^', 1) + if "^" in spec: + spec = spec.replace("^", f"+{mpi_gpu} ^", 1) else: spec = f"{spec} +{mpi_gpu}" From 38c740472fea8ad1ca8397db2addaffadc7a0777 Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Wed, 12 Feb 2025 17:09:31 +0100 Subject: [PATCH 04/17] Add first cut of mpi doc changes to demonstrate cxi use --- docs/recipes.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/recipes.md b/docs/recipes.md index 8cf715fc..dcc925d9 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -144,10 +144,12 @@ For example, in the recipe below, only `netcdf-fortran` will be built with the ` ### MPI -Stackinator can configure cray-mpich (CUDA, ROCM, or non-GPU aware) on a per-environment basis, by setting the `mpi` field in an environment. +Stackinator can configure cray-mpich (CUDA, ROCM, or non-GPU aware) or OpenMPI (with or without CUDA) (on a per-environment basis, by setting the `mpi` field in an environment. !!! note - Future versions of Stackinator will support OpenMPI, MPICH and MVAPICH when (and if) they develop robust support for HPE SlingShot 11 interconnect. + Future versions of Stackinator will fully support OpenMPI, MPICH and MVAPICH when (and if) they develop robust support for HPE SlingShot 11 interconnect. + + Current OpenMPI support has been tested lightly and is not guaranteed to be production ready - only OpenMPI@5.x.x is supported (default is @5.0.6 at the time of writing) - CUDA is supported, ROCM has not yet been tested. If the `mpi` field is not set, or is set to `null`, MPI will not be configured in an environment: ```yaml title="environments.yaml: no MPI" @@ -157,12 +159,18 @@ serial-env: ``` To configure MPI without GPU support, set the `spec` field with an optional version: -```yaml title="environments.yaml: MPI without GPU support" +```yaml title="environments.yaml: Cray-mpich without GPU support" host-env: mpi: spec: cray-mpich@8.1.23 # ... ``` +```yaml title="environments.yaml: OpenMPI without GPU support" +host-env: + mpi: + spec: openmpi + # ... +``` GPU-aware MPI can be configured by setting the optional `gpu` field to specify whether to support `cuda` or `rocm` GPUs: ```yaml title="environments.yaml: GPU aware MPI" @@ -176,7 +184,22 @@ rocm-env: spec: cray-mpich gpu: rocm # ... +ompi-cuda-env: + mpi: + spec: openmpi + gpu: cuda + # ... +``` +#### Experimental libfabric 2.x support with cray-mpich +HPE recently open-sourced the libfabric/cxi provider (and related drivers) and this can be built into cray-mpich by adding the `+cxi` variant to the spec +```yaml title="environments.yaml: MPI using new libfabric/cxi stack" +mpich-cxi-env: + mpi: + spec: cray-mpich +cxi + gpu: cuda + # ... ``` +OpenMPI does not provide a `cxi` option since it is mandatory to use it for builds on the alps cluster. Currently the performance of OpenMPI on Alps clusters might not be optimal and work is ongoing to fine tune it especially for intra-node performance. !!! alps @@ -207,7 +230,7 @@ The list of software packages to install is configured in the `spec:` field of a The `deprecated: ` field controls if Spack should consider versions marked as deprecated, and can be set to `true` or `false` (for considering or not considering deprecated versions, respectively). The `unify:` field controls the Spack concretiser, and can be set to three values `true`, `false` or `when_possible`. -The +The ```yaml cuda-env: From ac66c03489de09b9fa16fdc652dea4d7fab6937b Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Thu, 13 Feb 2025 00:31:13 +0100 Subject: [PATCH 05/17] Fix spec parsing to allow "cray-mpich +cxi" type strings --- stackinator/recipe.py | 47 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 68d91887..051a3159 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -3,6 +3,7 @@ import jinja2 import yaml +import re from . import cache, root_logger, schema, spack_util @@ -297,32 +298,32 @@ def generate_environment_specs(self, raw): mpi = config["mpi"] mpi_spec = mpi["spec"] mpi_gpu = mpi["gpu"] + if mpi_spec: - try: - mpi_impl, mpi_ver = mpi_spec.strip().split(sep="@", maxsplit=1) - except ValueError: - mpi_impl = mpi_spec.strip() - mpi_ver = None - - if mpi_impl in Recipe.valid_mpi_specs: - default_ver, options = Recipe.valid_mpi_specs[mpi_impl] - if mpi_ver: - version_opt = f"@{mpi_ver}" + # split mpi_spec into mpi_impl, mpi_ver, mpi_variants using 3 capture groups (2 optional) + pattern = r"([^\s@~]+)(?:\s*@(\S+))?(?:([\s\+~].+))?" + match = re.match(pattern, mpi_spec.strip()) + if match: + mpi_impl = match.group(1) + mpi_ver = match.group(2).strip() if match.group(2) else None + mpi_variants = match.group(3).strip() if match.group(3) else None + if mpi_impl in Recipe.valid_mpi_specs: + default_ver, options = Recipe.valid_mpi_specs[mpi_impl] + version_opt = f"@{mpi_ver}" if mpi_ver else f"@{default_ver}" if default_ver else "" + spec = f"{mpi_impl}{version_opt} {mpi_variants or options or ''}".strip() + + if mpi_gpu: + if "^" in spec: + spec = spec.replace("^", f"+{mpi_gpu} ^", 1) + else: + spec = f"{spec} +{mpi_gpu}" + + environments[name]["specs"].append(spec) else: - version_opt = f"@{default_ver}" if default_ver else "" - - spec = f"{mpi_impl}{version_opt} {options or ''}".strip() - - if mpi_gpu: - if "^" in spec: - spec = spec.replace("^", f"+{mpi_gpu} ^", 1) - else: - spec = f"{spec} +{mpi_gpu}" - - environments[name]["specs"].append(spec) + # TODO: Create a custom exception type + raise Exception(f"Unsupported mpi: {mpi_impl}") else: - # TODO: Create a custom exception type - raise Exception(f"Unsupported mpi: {mpi_impl}") + raise ValueError(f"Invalid mpi_spec format: {mpi_spec}") # set constraints that ensure the the main compiler is always used to build packages # that do not explicitly request a compiler. From bee3df93668a36fa49e45dd98b32a4fe4703a2bf Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Tue, 18 Feb 2025 15:45:05 +0100 Subject: [PATCH 06/17] Extend mpi syntax to allow specs and defaults from networking.yaml --- stackinator/builder.py | 43 ++++++++++++++++++++-- stackinator/recipe.py | 53 ++++------------------------ stackinator/schema/environments.json | 5 +-- 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 61d0e3ad..3b26757a 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -353,21 +353,60 @@ def generate(self, recipe): with dst.open("w") as fid: fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + packages_data = {} + # append network packages to packages.yaml + network_config_path = system_config_path / "network.yaml" + if not network_config_path.is_file(): + raise FileNotFoundError(f"The network configuration file '{network_config_path}' does not exist") + with network_config_path.open() as fid: + network_config = yaml.load(fid, Loader=yaml.Loader) + packages_data.update(network_config["packages"]) + # append recipe packages to packages.yaml if recipe.packages: system_packages = system_config_path / "packages.yaml" - packages_data = {} if system_packages.is_file(): # load system yaml with system_packages.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) - packages_data = raw["packages"] + packages_data.update(raw["packages"]) packages_data.update(recipe.packages["packages"]) packages_yaml = yaml.dump({"packages": packages_data}) packages_path = config_path / "packages.yaml" with packages_path.open("w") as fid: fid.write(packages_yaml) + # validate the recipe mpi selection + for name, config in recipe.environments.items(): + if config["mpi"]: + mpi = config["mpi"] + mpi_spec = mpi["spec"] + mpi_gpu = mpi["gpu"] + mpi_xspec = mpi["xspec"] if "xspec" in mpi else None + + if mpi_spec: + try: + mpi_impl, mpi_ver = mpi_spec.strip().split(sep="@", maxsplit=1) + except ValueError: + mpi_impl = mpi_spec.strip() + mpi_ver = None + + if mpi_impl in network_config["mpi_supported"]: + default_ver = network_config[mpi_impl]["version"] + default_spec = network_config[mpi_impl]["spec"] if "spec" in network_config[mpi_impl] else "" + # select users version or the default version if user did not specify + version_opt = f"@{mpi_ver or default_ver}" + # create full spec based on user provided spec or default spec + spec_opt = f"{mpi_impl}{version_opt} {mpi_xspec or default_spec}" + if mpi_gpu and not mpi_gpu in spec_opt: + spec_opt = f"{spec_opt} +{mpi_gpu}" + recipe.environments[name]["specs"].append(spec_opt) + else: + # TODO: Create a custom exception type + raise Exception(f"Unsupported mpi: {mpi_impl}") + print(f"RECIPE spec: {recipe.environments[name]['specs']}") + + # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to # build the stack, and then be part of the upstream provided diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 051a3159..f533957b 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -9,18 +9,6 @@ class Recipe: - valid_mpi_specs = { - "cray-mpich": (None, None), - "mpich": ("4.1", "device=ch4 netmod=ofi +slurm"), - "mvapich2": ( - "3.0a", - "+xpmem fabrics=ch4ofi ch4_max_vcis=4 process_managers=slurm", - ), - "openmpi": ( - "5", - "+internal-pmix fabrics=cma,ofi,xpmem schedulers=slurm +cray-xpmem ^libfabric@main ^libcxi@main ^cxi-driver@main ^cassini-headers@main", - ), - } @property def path(self): @@ -284,46 +272,19 @@ def generate_environment_specs(self, raw): for _, config in environments.items(): config["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - # check the environment descriptions and ammend where features are missing + # check the environment descriptions and amend where features are missing for name, config in environments.items(): if ("specs" not in config) or (config["specs"] is None): environments[name]["specs"] = [] if "mpi" not in config: - environments[name]["mpi"] = {"spec": None, "gpu": None} + environments[name]["mpi"] = {"spec": None, "gpu": None, "network": {"spec": None}} - # complete configuration of MPI in each environment - for name, config in environments.items(): - if config["mpi"]: - mpi = config["mpi"] - mpi_spec = mpi["spec"] - mpi_gpu = mpi["gpu"] - - if mpi_spec: - # split mpi_spec into mpi_impl, mpi_ver, mpi_variants using 3 capture groups (2 optional) - pattern = r"([^\s@~]+)(?:\s*@(\S+))?(?:([\s\+~].+))?" - match = re.match(pattern, mpi_spec.strip()) - if match: - mpi_impl = match.group(1) - mpi_ver = match.group(2).strip() if match.group(2) else None - mpi_variants = match.group(3).strip() if match.group(3) else None - if mpi_impl in Recipe.valid_mpi_specs: - default_ver, options = Recipe.valid_mpi_specs[mpi_impl] - version_opt = f"@{mpi_ver}" if mpi_ver else f"@{default_ver}" if default_ver else "" - spec = f"{mpi_impl}{version_opt} {mpi_variants or options or ''}".strip() - - if mpi_gpu: - if "^" in spec: - spec = spec.replace("^", f"+{mpi_gpu} ^", 1) - else: - spec = f"{spec} +{mpi_gpu}" - - environments[name]["specs"].append(spec) - else: - # TODO: Create a custom exception type - raise Exception(f"Unsupported mpi: {mpi_impl}") - else: - raise ValueError(f"Invalid mpi_spec format: {mpi_spec}") + if "network" not in config["mpi"]: + environments[name]["mpi"]["network"] = {"spec": ""} + + # we have not loaded the system configs yet, so mpi information will be generated + # during the builder phase. We will validate the mpi information then. # set constraints that ensure the the main compiler is always used to build packages # that do not explicitly request a compiler. diff --git a/stackinator/schema/environments.json b/stackinator/schema/environments.json index 9faefbd6..39f335bb 100644 --- a/stackinator/schema/environments.json +++ b/stackinator/schema/environments.json @@ -47,7 +47,8 @@ "gpu": { "enum": ["cuda", "rocm", null, false], "default": null - } + }, + "xspec": {"type": "string"} } }, {"enum": [null, false]} @@ -87,7 +88,7 @@ }, "uenv": { "type": "object", - "additionalPropertis": false, + "additionalProperties": false, "properties": { "add_compilers": { "type": "boolean" }, "prefix_paths": { From 30c98f1fd420ea9b4abaa6ea6f2ad1231e14d3ab Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Tue, 18 Feb 2025 17:25:35 +0100 Subject: [PATCH 07/17] Break xspec into xspec and depends to simplify customization --- stackinator/builder.py | 10 ++++++++-- stackinator/schema/environments.json | 10 +++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 3b26757a..10e47fb1 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -383,6 +383,7 @@ def generate(self, recipe): mpi_spec = mpi["spec"] mpi_gpu = mpi["gpu"] mpi_xspec = mpi["xspec"] if "xspec" in mpi else None + mpi_deps = mpi["depends"] if "depends" in mpi else None if mpi_spec: try: @@ -394,17 +395,22 @@ def generate(self, recipe): if mpi_impl in network_config["mpi_supported"]: default_ver = network_config[mpi_impl]["version"] default_spec = network_config[mpi_impl]["spec"] if "spec" in network_config[mpi_impl] else "" - # select users version or the default version if user did not specify + default_deps = network_config[mpi_impl]["depends"] if "depends" in network_config[mpi_impl] else "" + + # select users versions or the default versions if user did not specify version_opt = f"@{mpi_ver or default_ver}" # create full spec based on user provided spec or default spec spec_opt = f"{mpi_impl}{version_opt} {mpi_xspec or default_spec}" if mpi_gpu and not mpi_gpu in spec_opt: spec_opt = f"{spec_opt} +{mpi_gpu}" + deps_opt = mpi_deps or default_deps + for dep in deps_opt: + spec_opt = f"{spec_opt} ^{dep}" + recipe.environments[name]["specs"].append(spec_opt) else: # TODO: Create a custom exception type raise Exception(f"Unsupported mpi: {mpi_impl}") - print(f"RECIPE spec: {recipe.environments[name]['specs']}") # Add custom spack package recipes, configured via Spack repos. diff --git a/stackinator/schema/environments.json b/stackinator/schema/environments.json index 39f335bb..1c80c4a3 100644 --- a/stackinator/schema/environments.json +++ b/stackinator/schema/environments.json @@ -48,8 +48,13 @@ "enum": ["cuda", "rocm", null, false], "default": null }, - "xspec": {"type": "string"} - } + "xspec": {"type": "string"}, + "depends": { + "type": "array", + "items": {"type": "string"}, + "default": [] + } + } }, {"enum": [null, false]} ], @@ -110,4 +115,3 @@ } } } - From 0a11871939108127d4fc503994dbd35c2be764e9 Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Wed, 26 Feb 2025 14:49:50 +0100 Subject: [PATCH 08/17] Cleanup mpi spec logic, add log messages, move network packages into packages.yaml --- stackinator/builder.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 10e47fb1..2163aaef 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -360,7 +360,7 @@ def generate(self, recipe): raise FileNotFoundError(f"The network configuration file '{network_config_path}' does not exist") with network_config_path.open() as fid: network_config = yaml.load(fid, Loader=yaml.Loader) - packages_data.update(network_config["packages"]) + packages_data.update(network_config.get("packages", {})) # append recipe packages to packages.yaml if recipe.packages: @@ -378,41 +378,51 @@ def generate(self, recipe): # validate the recipe mpi selection for name, config in recipe.environments.items(): + # config[mpi] holds the user specified settings in environment.yaml if config["mpi"]: mpi = config["mpi"] - mpi_spec = mpi["spec"] - mpi_gpu = mpi["gpu"] - mpi_xspec = mpi["xspec"] if "xspec" in mpi else None - mpi_deps = mpi["depends"] if "depends" in mpi else None + user_mpi_spec = mpi["spec"] + user_mpi_gpu = mpi["gpu"] + user_mpi_xspec = mpi["xspec"] if "xspec" in mpi else None + user_mpi_deps = mpi["depends"] if "depends" in mpi else None + self._logger.debug( + f"User MPI selection: spec={user_mpi_spec}, gpu={user_mpi_gpu}, xspec={user_mpi_xspec}, deps={user_mpi_deps}" + ) - if mpi_spec: + if user_mpi_spec: try: - mpi_impl, mpi_ver = mpi_spec.strip().split(sep="@", maxsplit=1) + mpi_impl, mpi_ver = user_mpi_spec.strip().split(sep="@", maxsplit=1) except ValueError: - mpi_impl = mpi_spec.strip() + mpi_impl = user_mpi_spec.strip() mpi_ver = None + # network_config holds the system specified settings in cluster config / network.yaml if mpi_impl in network_config["mpi_supported"]: default_ver = network_config[mpi_impl]["version"] default_spec = network_config[mpi_impl]["spec"] if "spec" in network_config[mpi_impl] else "" - default_deps = network_config[mpi_impl]["depends"] if "depends" in network_config[mpi_impl] else "" + default_deps = ( + network_config[mpi_impl]["depends"] if "depends" in network_config[mpi_impl] else "" + ) + self._logger.debug( + f"System MPI selection: spec={mpi_impl}@{default_ver} {default_spec}, deps={default_deps}" + ) # select users versions or the default versions if user did not specify version_opt = f"@{mpi_ver or default_ver}" # create full spec based on user provided spec or default spec - spec_opt = f"{mpi_impl}{version_opt} {mpi_xspec or default_spec}" - if mpi_gpu and not mpi_gpu in spec_opt: - spec_opt = f"{spec_opt} +{mpi_gpu}" - deps_opt = mpi_deps or default_deps + spec_opt = f"{mpi_impl}{version_opt} {user_mpi_xspec or default_spec}" + if user_mpi_gpu and user_mpi_gpu not in spec_opt: + spec_opt = f"{spec_opt} +{user_mpi_gpu}" + deps_opt = user_mpi_deps or default_deps for dep in deps_opt: spec_opt = f"{spec_opt} ^{dep}" + self._logger.debug(f"Final MPI selection: spec={spec_opt}") recipe.environments[name]["specs"].append(spec_opt) else: # TODO: Create a custom exception type raise Exception(f"Unsupported mpi: {mpi_impl}") - # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to # build the stack, and then be part of the upstream provided From 2b342a7323c1876a767eae8c82eda08bad40acdc Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Wed, 26 Feb 2025 15:27:48 +0100 Subject: [PATCH 09/17] Fix style CI failures --- stackinator/builder.py | 3 ++- stackinator/recipe.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2163aaef..61929da0 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -386,7 +386,8 @@ def generate(self, recipe): user_mpi_xspec = mpi["xspec"] if "xspec" in mpi else None user_mpi_deps = mpi["depends"] if "depends" in mpi else None self._logger.debug( - f"User MPI selection: spec={user_mpi_spec}, gpu={user_mpi_gpu}, xspec={user_mpi_xspec}, deps={user_mpi_deps}" + f"User MPI selection: spec={user_mpi_spec}, gpu={user_mpi_gpu}, " + f"xspec={user_mpi_xspec}, deps={user_mpi_deps}" ) if user_mpi_spec: diff --git a/stackinator/recipe.py b/stackinator/recipe.py index f533957b..4f5b565b 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -3,7 +3,6 @@ import jinja2 import yaml -import re from . import cache, root_logger, schema, spack_util From baf6876bd65331cb5e86945c7835a04cc97edbe1 Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Wed, 26 Feb 2025 15:33:36 +0100 Subject: [PATCH 10/17] Add mpi: depends to schema unit test --- unittests/test_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 9bc03264..37d0b326 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -133,7 +133,7 @@ def test_environments_yaml(yaml_path): # test defaults were set correctly assert env["unify"] == "when_possible" assert env["packages"] == ["perl", "git"] - assert env["mpi"] == {"spec": "cray-mpich", "gpu": "cuda"} + assert env["mpi"] == {"depends": [], "spec": "cray-mpich", "gpu": "cuda"} assert env["variants"] == ["+mpi", "+cuda"] assert env["views"] == {"default": None} From 493246b9a733249ea32e4fa4459888cc5918da25 Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Tue, 4 Mar 2025 11:12:48 +0100 Subject: [PATCH 11/17] Update docs to reflect recent mpi syntax changes --- docs/recipes.md | 55 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/recipes.md b/docs/recipes.md index dcc925d9..e68e7e2b 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -146,10 +146,10 @@ For example, in the recipe below, only `netcdf-fortran` will be built with the ` Stackinator can configure cray-mpich (CUDA, ROCM, or non-GPU aware) or OpenMPI (with or without CUDA) (on a per-environment basis, by setting the `mpi` field in an environment. -!!! note +!!! note Note (on non default MPI veresions) Future versions of Stackinator will fully support OpenMPI, MPICH and MVAPICH when (and if) they develop robust support for HPE SlingShot 11 interconnect. - Current OpenMPI support has been tested lightly and is not guaranteed to be production ready - only OpenMPI@5.x.x is supported (default is @5.0.6 at the time of writing) - CUDA is supported, ROCM has not yet been tested. + Current OpenMPI support has been tested lightly and is not guaranteed to be production ready - only OpenMPI@5.x.x is supported (default is @5.0.6 at the time of writing - 2025.03.04) - CUDA is supported, ROCM has not yet been tested. If the `mpi` field is not set, or is set to `null`, MPI will not be configured in an environment: ```yaml title="environments.yaml: no MPI" @@ -191,16 +191,57 @@ ompi-cuda-env: # ... ``` #### Experimental libfabric 2.x support with cray-mpich -HPE recently open-sourced the libfabric/cxi provider (and related drivers) and this can be built into cray-mpich by adding the `+cxi` variant to the spec -```yaml title="environments.yaml: MPI using new libfabric/cxi stack" +HPE has open-sourced the libfabric/cxi provider (and related drivers) and these can be built into cray-mpich by adding a dependency to a newer version of libfabric. +The system default is libfabric@1.15.2 - which can be changed by adding a `depends` field to the yaml. A non default version (newer than 1.15.2) will trigger a build of libfabric using libcxi, cxi-driver, cassini-headers). + +This syntax appplies to both mpich and openmpi builds. +```yaml title="environments1.yaml: cray-mpich using new libfabric/cxi stack" mpich-cxi-env: mpi: - spec: cray-mpich +cxi + spec: cray-mpich gpu: cuda - # ... + depends: [libfabric@main] + # on release of libfabric@2, we recommended using @2 in preference to @main ... +``` +```yaml title="environments2.yaml: openmpi using new libfabric/cxi stack" +openmpi-cxi-env: + mpi: + spec: openmpi + gpu: cuda + depends: [libfabric@main] + # on release of libfabric@2, we recommended using @2 in preference to @main ... ``` -OpenMPI does not provide a `cxi` option since it is mandatory to use it for builds on the alps cluster. Currently the performance of OpenMPI on Alps clusters might not be optimal and work is ongoing to fine tune it especially for intra-node performance. +!!! note Note (on openmpi performance) + Currently the performance of OpenMPI on Alps clusters might not be optimal and work is ongoing to fine tune it especially for intra-node performance. + +#### Custom MPI builds +If an experimental version of OpenMPI is required, the yaml syntax supports additional options to enable this. the `xspec` tag may be used to supply extra spack `spec` options. +To illustrate usage, consider this example: a build of openmpi using a particular git branch named `mpi-continue-5.0.6` which in this case has a variant `+continuations` that is not available on the `main` or released branches. +The `xspec` tag allows the user to supply arbitrary spack variants and options that replace the defaults used by stackinator in its absence. +```yaml title="custom-openmpi.yaml: custom build of openmpi" +openmpi-custom-env: + mpi: + spec: openmpi@git.mpi-continue-5.0.6=main + xspec: +continuations +internal-pmix fabrics=cma,ofi,xpmem schedulers=slurm +cray-xpmem + gpu: cuda + depends: ["libfabric@main"] +``` +In this example, we must tell spack to fetch our custom git branch from a repo that is different from the default openmpi github version, by adding the following to our recipe `packages.yaml` +```yaml title="custom-packages.yaml: custom repo for openmpi" + # mpi-continue-5.0.6 branch available from here + openmpi: + package_attributes: + git: https://github.com/your-username/ompi +``` + +!!! note Note (git tags) + To build using a specific git commit, use the syntax + ``` + spec: openmpi@git.9a4079916dd13d4190fe224102b57757789c13da=main + ``` +It is therefore possible to build arbitrary versions of MPI using custom options/branches etc using these combinations of settings. +### Version info !!! alps As new versions of cray-mpich are released with CPE, they are provided on Alps vClusters, via the Spack package repo in the [CSCS cluster configuration repo](https://github.com/eth-cscs/alps-cluster-config/tree/master/site/repo). From 03dfe2913e19d63607b630f546316c4640749af4 Mon Sep 17 00:00:00 2001 From: John Biddiscombe Date: Tue, 4 Mar 2025 11:59:06 +0100 Subject: [PATCH 12/17] Remove note titles as they break rendering sometimes --- docs/recipes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/recipes.md b/docs/recipes.md index e68e7e2b..1f5c2c1c 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -146,7 +146,7 @@ For example, in the recipe below, only `netcdf-fortran` will be built with the ` Stackinator can configure cray-mpich (CUDA, ROCM, or non-GPU aware) or OpenMPI (with or without CUDA) (on a per-environment basis, by setting the `mpi` field in an environment. -!!! note Note (on non default MPI veresions) +!!! note Future versions of Stackinator will fully support OpenMPI, MPICH and MVAPICH when (and if) they develop robust support for HPE SlingShot 11 interconnect. Current OpenMPI support has been tested lightly and is not guaranteed to be production ready - only OpenMPI@5.x.x is supported (default is @5.0.6 at the time of writing - 2025.03.04) - CUDA is supported, ROCM has not yet been tested. @@ -211,7 +211,7 @@ openmpi-cxi-env: depends: [libfabric@main] # on release of libfabric@2, we recommended using @2 in preference to @main ... ``` -!!! note Note (on openmpi performance) +!!! note Currently the performance of OpenMPI on Alps clusters might not be optimal and work is ongoing to fine tune it especially for intra-node performance. #### Custom MPI builds @@ -234,7 +234,7 @@ In this example, we must tell spack to fetch our custom git branch from a repo t git: https://github.com/your-username/ompi ``` -!!! note Note (git tags) +!!! note To build using a specific git commit, use the syntax ``` spec: openmpi@git.9a4079916dd13d4190fe224102b57757789c13da=main From f9dc46d7ad6fe98c9dced1253c726b908363d789 Mon Sep 17 00:00:00 2001 From: Ben Cumming Date: Tue, 18 Mar 2025 13:21:10 +0100 Subject: [PATCH 13/17] Update docs/recipes.md Co-authored-by: Mikael Simberg --- docs/recipes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/recipes.md b/docs/recipes.md index 1f5c2c1c..ac89558a 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -144,7 +144,7 @@ For example, in the recipe below, only `netcdf-fortran` will be built with the ` ### MPI -Stackinator can configure cray-mpich (CUDA, ROCM, or non-GPU aware) or OpenMPI (with or without CUDA) (on a per-environment basis, by setting the `mpi` field in an environment. +Stackinator can configure cray-mpich (CUDA, ROCM, or non-GPU aware) or OpenMPI (with or without CUDA) on a per-environment basis, by setting the `mpi` field in an environment. !!! note Future versions of Stackinator will fully support OpenMPI, MPICH and MVAPICH when (and if) they develop robust support for HPE SlingShot 11 interconnect. From a2dbb09291cf442dd80bd99d5867bd981eb1f644 Mon Sep 17 00:00:00 2001 From: Ben Cumming Date: Tue, 18 Mar 2025 13:21:22 +0100 Subject: [PATCH 14/17] Update docs/recipes.md Co-authored-by: Mikael Simberg --- docs/recipes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/recipes.md b/docs/recipes.md index ac89558a..a8a0cc52 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -159,7 +159,7 @@ serial-env: ``` To configure MPI without GPU support, set the `spec` field with an optional version: -```yaml title="environments.yaml: Cray-mpich without GPU support" +```yaml title="environments.yaml: Cray MPICH without GPU support" host-env: mpi: spec: cray-mpich@8.1.23 From a358f2e7f3c8dd6cb59dc058037230c37e6b4f33 Mon Sep 17 00:00:00 2001 From: Ben Cumming Date: Tue, 18 Mar 2025 13:21:38 +0100 Subject: [PATCH 15/17] Update docs/recipes.md Co-authored-by: Mikael Simberg --- docs/recipes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/recipes.md b/docs/recipes.md index a8a0cc52..d840e3ed 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -191,6 +191,7 @@ ompi-cuda-env: # ... ``` #### Experimental libfabric 2.x support with cray-mpich + HPE has open-sourced the libfabric/cxi provider (and related drivers) and these can be built into cray-mpich by adding a dependency to a newer version of libfabric. The system default is libfabric@1.15.2 - which can be changed by adding a `depends` field to the yaml. A non default version (newer than 1.15.2) will trigger a build of libfabric using libcxi, cxi-driver, cassini-headers). From 33a308dcce3f0677ce694fae03d29b82e81921ad Mon Sep 17 00:00:00 2001 From: Ben Cumming Date: Tue, 18 Mar 2025 13:22:35 +0100 Subject: [PATCH 16/17] Update docs/recipes.md Co-authored-by: Mikael Simberg --- docs/recipes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/recipes.md b/docs/recipes.md index d840e3ed..5c21eb94 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -216,6 +216,7 @@ openmpi-cxi-env: Currently the performance of OpenMPI on Alps clusters might not be optimal and work is ongoing to fine tune it especially for intra-node performance. #### Custom MPI builds + If an experimental version of OpenMPI is required, the yaml syntax supports additional options to enable this. the `xspec` tag may be used to supply extra spack `spec` options. To illustrate usage, consider this example: a build of openmpi using a particular git branch named `mpi-continue-5.0.6` which in this case has a variant `+continuations` that is not available on the `main` or released branches. The `xspec` tag allows the user to supply arbitrary spack variants and options that replace the defaults used by stackinator in its absence. From cca07c5374d23c7d701a6427eab32c7f63a40ff7 Mon Sep 17 00:00:00 2001 From: Ben Cumming Date: Tue, 18 Mar 2025 13:25:39 +0100 Subject: [PATCH 17/17] Update docs/recipes.md Co-authored-by: Mikael Simberg --- docs/recipes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/recipes.md b/docs/recipes.md index 5c21eb94..26db4a5f 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -244,6 +244,7 @@ In this example, we must tell spack to fetch our custom git branch from a repo t It is therefore possible to build arbitrary versions of MPI using custom options/branches etc using these combinations of settings. ### Version info + !!! alps As new versions of cray-mpich are released with CPE, they are provided on Alps vClusters, via the Spack package repo in the [CSCS cluster configuration repo](https://github.com/eth-cscs/alps-cluster-config/tree/master/site/repo).