Merge branch '2023.06-software.eessi.io' of github.com:EESSI/software-layer into ci_enable_sapphire_rapids

Author: bedroge

Diff for: .github/workflows/test-software.eessi.io.yml

@@ -60,20 +60,6 @@ jobs:
               # first check the CPU-only builds for this CPU target
               echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)"
               for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
-                  if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then
-                      if grep -q 2022b <<<"${easystack_file}"; then
-                        # skip the check of installed software on zen4 for foss/2022b builds
-                        continue
-                      fi
-                      if [[ $easystack_file == *"rebuilds"* ]]; then
-                        # Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that
-                        new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml)
-                        # first clean out the options then clean out the .eb name
-                        sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack
-                        diff --unified=0 "$easystack_file" "$new_easystack" || :
-                        easystack_file="$new_easystack"
-                      fi
-                  fi
                   echo "check missing installations for ${easystack_file}..."
                   ./check_missing_installations.sh ${easystack_file}
                   ec=$?

Diff for: EESSI-install-software.sh

@@ -271,11 +271,18 @@ fi
 
 # Install NVIDIA drivers in host_injections (if they exist)
 if command_exists "nvidia-smi"; then
-    echo "Command 'nvidia-smi' found. Installing NVIDIA drivers for use in prefix shell..."
-    ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
+    nvidia-smi --version
+    ec=$?
+    if [ ${ec} -eq 0 ]; then 
+        echo "Command 'nvidia-smi' found. Installing NVIDIA drivers for use in prefix shell..."
+        ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
+    else
+        echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully."
+        echo "This script now assumes this is NOT a GPU node."
+        echo "If, and only if, the current node actually does contain Nvidia GPUs, this should be considered an error."
+    fi
 fi
 
-
 if [ ! -z "${shared_fs_path}" ]; then
     shared_eb_sourcepath=${shared_fs_path}/easybuild/sources
     echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path"

Diff for: EESSI-remove-software.sh

@@ -116,9 +116,12 @@ if [ $EUID -eq 0 ]; then
             if [ -f ${easystack_file} ]; then
                 echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
                 # we need to remove existing installation directories first,
-                # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
+                # so let's figure out which modules have to be rebuilt by doing a
+                # dry-run and grepping "someapp/someversion" for the relevant
+                # lines (with [R] or [F])
+                #  * [F] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
                 #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-                rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+                rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[[FR]\]" | grep -o "module: .*[^)]" | awk '{print $2}')
                 for app in ${rebuild_apps}; do
                     # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all:
                     app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://')
@@ -130,13 +133,13 @@ if [ $EUID -eq 0 ]; then
                     echo_yellow "Removing ${app_dir} and ${app_module}..."
                     rm -rf ${app_dir}
                     rm -rf ${app_module}
-                    # recreate the installation directories and first-level subdirectories to work around permission denied
-                    # issues when rebuilding the package (see https://github.com/EESSI/software-layer/issues/556)
+                    # recreate the installation directory and do an ls on the first-level subdirectories to work around
+                    # permission issues when reinstalling the application (see https://github.com/EESSI/software-layer/issues/556)
                     echo_yellow "Recreating an empty ${app_dir}..."
                     mkdir -p ${app_dir}
-                    for app_subdir in ${app_subdirs}; do
-                        mkdir -p ${app_subdir}
-                    done
+                    # these subdirs don't (and shouldn't) exist, but we need to do the ls anyway as a workaround,
+                    # so redirect to /dev/null and ignore the exit code
+                    ls ${app_subdirs} >& /dev/null || true
                 done
             else
                 fatal_error "Easystack file ${easystack_file} not found!"

Diff for: bot/build.sh

@@ -90,11 +90,15 @@ if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR
 fi
 
-echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
-# replace any env variable in ${LOCAL_TMP} with its
-#   current value (e.g., a value that is local to the job)
-STORAGE=$(envsubst <<< ${LOCAL_TMP})
-echo "'${STORAGE}'"
+if [[ -z "${TMPDIR}" ]]; then
+    echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
+    # replace any env variable in ${LOCAL_TMP} with its
+    #   current value (e.g., a value that is local to the job)
+    STORAGE=$(envsubst <<< ${LOCAL_TMP})
+else
+    STORAGE=${TMPDIR}
+fi
+echo "bot/build.sh: STORAGE='${STORAGE}'"
 
 # make sure ${STORAGE} exists
 mkdir -p ${STORAGE}
@@ -115,7 +119,8 @@ mkdir -p ${SINGULARITY_TMPDIR}
 
 # load modules if LOAD_MODULES is not empty
 if [[ ! -z ${LOAD_MODULES} ]]; then
-    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    IFS=',' read -r -a modules <<< "$(echo "${LOAD_MODULES}")"
+    for mod in "${modules[@]}";
     do
         echo "bot/build.sh: loading module '${mod}'"
         module load ${mod}
@@ -243,14 +248,28 @@ mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
 # prepare arguments to eessi_container.sh specific to build step
 BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
+
 # add options required to handle NVIDIA support
 if command_exists "nvidia-smi"; then
-    echo "Command 'nvidia-smi' found, using available GPU"
-    BUILD_STEP_ARGS+=("--nvidia" "all")
+    # Accept that this may fail
+    set +e
+    nvidia-smi --version
+    ec=$?
+    set -e
+    if [ ${ec} -eq 0 ]; then
+        echo "Command 'nvidia-smi' found, using available GPU"
+        BUILD_STEP_ARGS+=("--nvidia" "all")
+    else
+        echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully."
+        echo "This script now assumes this is NOT a GPU node."
+        echo "If, and only if, the current node actually does contain Nvidia GPUs, this should be considered an error."
+        BUILD_STEP_ARGS+=("--nvidia" "install")
+    fi
 else
     echo "No 'nvidia-smi' found, no available GPU but allowing overriding this check"
     BUILD_STEP_ARGS+=("--nvidia" "install")
 fi
+
 # Retain location for host injections so we don't reinstall CUDA
 # (Always need to run the driver installation as available driver may change)
 if [[ ! -z ${SHARED_FS_PATH} ]]; then