Skip to content

Commit b8946ee

Browse files
bedrogebedroge
authored
Merge pull request #667 from TopRichard/eessi-2023.06-EESSI_Module_bash
EESSI bash initialization to module file
2 parents eb11ea2 + 47959c5 commit b8946ee

File tree

3 files changed

+161
-0
lines changed

3 files changed

+161
-0
lines changed

‎.github/workflows/tests_eessi_module.yml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
2+
name: Tests for eessi_module_functionality in software.eessi.io
3+
on:
4+
push:
5+
branches: [ "*-software.eessi.io" ]
6+
pull_request:
7+
permissions:
8+
contents: read # to fetch code (actions/checkout)
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest
12+
strategy:
13+
fail-fast: false
14+
matrix:
15+
EESSI_VERSION:
16+
- 2023.06
17+
steps:
18+
- name: Check out software-layer repository
19+
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
20+
21+
- name: Mount EESSI CernVM-FS pilot repository
22+
uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
23+
with:
24+
cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
25+
cvmfs_http_proxy: DIRECT
26+
cvmfs_repositories: software.eessi.io
27+
28+
- name: Test for making sure spider cache is being used and not being rebuilt
29+
run: |
30+
. /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod
31+
export MODULEPATH=init/modules
32+
configfile="configfile.txt"
33+
module -T load EESSI/${{matrix.EESSI_VERSION}}
34+
module --config > "${configfile}" 2>&1
35+
grep cache "${configfile}" | grep software | grep -v compat
36+
if timeout 10s bash -c "LMOD_PAGER=none module --terse avail" && grep cache "${configfile}" | grep software | grep -v compat; then
37+
echo "EESSI spider cache is being used"
38+
else
39+
echo "EESSI spider cache is being rebuilt" >&2
40+
exit 1
41+
fi
42+
env | grep LMOD
43+
module purge
44+
unset MODULEPATH
45+
46+
- name: Test for archdetect_cpu functionality with invalid path
47+
run: |
48+
. /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod
49+
export MODULEPATH=init/modules
50+
set +e # Do not exit immediately if a command exits with a non-zero status
51+
export EESSI_ARCHDETECT_OPTIONS="dummy/cpu"
52+
outfile="outfile.txt"
53+
module load EESSI/${{matrix.EESSI_VERSION}} > "${outfile}" 2>&1
54+
cat "${outfile}"
55+
if grep -q "Software directory check" "${outfile}"; then
56+
echo "Test for picking up invalid path on \${archdetect_cpu} PASSED"
57+
else
58+
echo "Test for picking up invalid path on \${archdetect_cpu} FAILED" >&2
59+
exit 1
60+
fi
61+
unset EESSI_ARCHDETECT_OPTIONS
62+
set -e # Re-enable exit on non-zero status
63+
64+
- name: Test for expected variables while adding dummy cpu archs and loading EESSI module
65+
run: |
66+
. /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod
67+
export MODULEPATH=init/modules
68+
CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath)
69+
export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1"
70+
moduleoutfile="moduleout.txt"
71+
sourceoutfile="sourceout.txt"
72+
module load EESSI/${{matrix.EESSI_VERSION}}
73+
env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${moduleoutfile}"
74+
module unload EESSI/${{matrix.EESSI_VERSION}}
75+
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
76+
env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${sourceoutfile}"
77+
cat "${moduleoutfile}"
78+
cat "${sourceoutfile}"
79+
if (diff "${moduleoutfile}" "${sourceoutfile}" > /dev/null); then
80+
echo "Test for checking env variables PASSED"
81+
else
82+
echo "Test for checking env variables FAILED" >&2
83+
exit 1
84+
fi
85+

‎init/modules/EESSI/2023.06.lua

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
help([[
2+
Description
3+
===========
4+
The European Environment for Scientific Software Installations (EESSI, pronounced as easy) is a collaboration between different European partners in HPC community.The goal of this project is to build a common stack of scientific software installations for HPC systems and beyond, including laptops, personal workstations and cloud infrastructure.
5+
6+
More information
7+
================
8+
- URL: https://www.eessi.io/docs/
9+
]])
10+
whatis("Description: The European Environment for Scientific Software Installations (EESSI, pronounced as easy) is a collaboration between different European partners in HPC community. The goal of this project is to build a common stack of scientific software installations for HPC systems and beyond, including laptops, personal workstations and cloud infrastructure.")
11+
whatis("URL: https://www.eessi.io/docs/")
12+
conflict("EESSI")
13+
local eessi_version = myModuleVersion()
14+
local eessi_repo = "/cvmfs/software.eessi.io"
15+
local eessi_prefix = pathJoin(eessi_repo, "versions", eessi_version)
16+
local eessi_os_type = "linux"
17+
setenv("EESSI_VERSION", eessi_version)
18+
setenv("EESSI_CVMFS_REPO", eessi_repo)
19+
setenv("EESSI_OS_TYPE", eessi_os_type)
20+
function archdetect_cpu()
21+
local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper.sh')
22+
if not os.getenv("EESSI_ARCHDETECT_OPTIONS") then
23+
if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then
24+
LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ARCHDETECT_OPTIONS to the available cpu architecture in the form of: x86_64/intel/haswell:x86_64/generic or aarch64/neoverse_v1:aarch64/generic")
25+
end
26+
source_sh("bash", script)
27+
end
28+
local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or ""
29+
for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do
30+
if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then
31+
-- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected,
32+
-- since optimized software installations for Zen4 are a work-in-progress,
33+
-- see https://gitlab.com/eessi/support/-/issues/37
34+
if archdetect_filter_cpu == "x86_64/amd/zen4" then
35+
archdetect_filter_cpu = "x86_64/amd/zen3"
36+
if mode() == "load" then
37+
LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.")
38+
end
39+
end
40+
return archdetect_filter_cpu
41+
end
42+
end
43+
LmodError("Software directory check for the detected architecture failed")
44+
end
45+
local archdetect = archdetect_cpu()
46+
local eessi_cpu_family = archdetect:match("([^/]+)")
47+
local eessi_software_subdir = archdetect
48+
local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family)
49+
local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir)
50+
local eessi_module_path = pathJoin(eessi_software_path, "modules", "all")
51+
local eessi_site_module_path = string.gsub(eessi_module_path, "versions", "host_injections")
52+
setenv("EPREFIX", eessi_eprefix)
53+
setenv("EESSI_CPU_FAMILY", eessi_cpu_family)
54+
setenv("EESSI_SITE_MODULEPATH", eessi_site_module_path)
55+
setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir)
56+
setenv("EESSI_PREFIX", eessi_prefix)
57+
setenv("EESSI_EPREFIX", eessi_eprefix)
58+
prepend_path("PATH", pathJoin(eessi_eprefix, "bin"))
59+
prepend_path("PATH", pathJoin(eessi_eprefix, "usr/bin"))
60+
setenv("EESSI_SOFTWARE_PATH", eessi_software_path)
61+
setenv("EESSI_MODULEPATH", eessi_module_path)
62+
if ( mode() ~= "spider" ) then
63+
prepend_path("MODULEPATH", eessi_module_path)
64+
end
65+
prepend_path("LMOD_RC", pathJoin(eessi_software_path, "/.lmod/lmodrc.lua"))
66+
prepend_path("MODULEPATH", eessi_site_module_path)
67+
setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod"))
68+
if mode() == "load" then
69+
LmodMessage("EESSI/" .. eessi_version .. " loaded successfully")
70+
end

‎install_scripts.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,12 @@ mc_files=(
102102
)
103103
copy_files_by_list ${TOPDIR}/init/Magic_Castle ${INSTALL_PREFIX}/init/Magic_Castle "${mc_files[@]}"
104104

105+
# Copy for init/modules/EESSI directory
106+
mc_files=(
107+
2023.06.lua
108+
)
109+
copy_files_by_list ${TOPDIR}/init/modules/EESSI ${INSTALL_PREFIX}/init/modules/EESSI "${mc_files[@]}"
110+
105111
# Copy for the scripts directory
106112
script_files=(
107113
utils.sh

0 commit comments

Comments
 (0)