Merge remote-tracking branch 'upstream/main' into adjustments-for-eessi-bot

Merge updates to build_container.sh

Author: truib

.github/workflows/tests_scripts.yml

@@ -6,6 +6,8 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
+      - utils.sh
+      - update_lmod_cache.sh
 
   pull_request:
     branches:
@@ -14,6 +16,8 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
+      - utils.sh
+      - update_lmod_cache.sh
 jobs:
   build:
     runs-on: ubuntu-20.04
@@ -25,8 +29,9 @@ jobs:
       run: |
         sudo apt-get install alien
         alien --version
-        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/singularity-3.8.5-2.el8.x86_64.rpm
-        sudo alien -d singularity-3.8.5-2.el8.x86_64.rpm
+        singularity_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*\(singularity[0-9._a-z-]*.rpm\).*/\1/g')
+        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/$singularity_rpm
+        sudo alien -d $singularity_rpm
         sudo apt install ./singularity*.deb
         singularity --version
 

EESSI-pilot-install-software.sh

@@ -5,34 +5,7 @@
 
 TOPDIR=$(dirname $(realpath $0))
 
-function echo_green() {
-    echo -e "\e[32m$1\e[0m"
-}
-
-function echo_red() {
-    echo -e "\e[31m$1\e[0m"
-}
-
-function echo_yellow() {
-    echo -e "\e[33m$1\e[0m"
-}
-
-function fatal_error() {
-    echo_red "ERROR: $1" >&2
-    exit 1
-}
-
-function check_exit_code {
-    ec=$1
-    ok_msg=$2
-    fail_msg=$3
-
-    if [[ $ec -eq 0 ]]; then
-        echo_green "${ok_msg}"
-    else
-        fatal_error "${fail_msg}"
-    fi
-}
+source $TOPDIR/utils.sh
 
 # honor $TMPDIR if it is already defined, use /tmp otherwise
 if [ -z $TMPDIR ]; then
@@ -339,16 +312,7 @@ if [ ! -f $LMOD_RC ]; then
     check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC"
 fi
 
-# we need to specify the path to the Lmod cache dir + timestamp file to ensure
-# that update_lmod_system_cache_files updates correct Lmod cache
-lmod_cache_dir=${EASYBUILD_INSTALLPATH}/.lmod/cache
-lmod_cache_timestamp_file=${EASYBUILD_INSTALLPATH}/.lmod/cache/timestamp
-modpath=${EASYBUILD_INSTALLPATH}/modules/all
-
-${LMOD_DIR}/update_lmod_system_cache_files -d ${lmod_cache_dir} -t ${lmod_cache_timestamp_file} ${modpath}
-check_exit_code $? "Lmod cache updated" "Lmod cache update failed!"
-
-ls -lrt ${EASYBUILD_INSTALLPATH}/.lmod/cache
+$TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
 
 echo ">> Checking for missing installations..."
 ok_msg="No missing installations, party time!"

build_container.sh

@@ -41,7 +41,15 @@ mkdir -p $EESSI_TMPDIR/{home,overlay-upper,overlay-work}
 mkdir -p $EESSI_TMPDIR/{var-lib-cvmfs,var-run-cvmfs}
 # configure Singularity
 export SINGULARITY_CACHEDIR=$EESSI_TMPDIR/singularity_cache
-export SINGULARITY_BIND="$EESSI_TMPDIR/var-run-cvmfs:/var/run/cvmfs,$EESSI_TMPDIR/var-lib-cvmfs:/var/lib/cvmfs,$EESSI_TMPDIR"
+
+# take into account that $SINGULARITY_BIND may be defined already, to bind additional paths into the build container
+BIND_PATHS="$EESSI_TMPDIR/var-run-cvmfs:/var/run/cvmfs,$EESSI_TMPDIR/var-lib-cvmfs:/var/lib/cvmfs,$EESSI_TMPDIR"
+if [ -z $SINGULARITY_BIND ]; then
+    export SINGULARITY_BIND="$BIND_PATHS"
+else
+    export SINGULARITY_BIND="$SINGULARITY_BIND,$BIND_PATHS"
+fi
+
 export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER"
 
 # set environment variables for fuse mounts in Singularity container

update_lmod_cache.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Script to update Lmod cache in EESSI
+#
+
+TOPDIR=$(dirname $(realpath $0))
+
+source $TOPDIR/utils.sh
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 <path to compat layer directory> <path to software installation prefix>" >&2
+    exit 1
+fi
+EPREFIX=$1
+EASYBUILD_INSTALLPATH=$2
+
+if [ ! -d $EPREFIX ]; then
+    echo "\$EPREFIX does not exist!" >&2
+    exit 2
+fi
+if [ ! -d $EASYBUILD_INSTALLPATH ]; then
+    echo "\$EASYBUILD_INSTALLPATH does not exist!" >&2
+    exit 2
+fi
+
+source $EPREFIX/usr/share/Lmod/init/bash
+
+# we need to specify the path to the Lmod cache dir + timestamp file to ensure
+# that update_lmod_system_cache_files updates correct Lmod cache
+lmod_cache_dir=${EASYBUILD_INSTALLPATH}/.lmod/cache
+lmod_cache_timestamp_file=${EASYBUILD_INSTALLPATH}/.lmod/cache/timestamp
+modpath=${EASYBUILD_INSTALLPATH}/modules/all
+
+${LMOD_DIR}/update_lmod_system_cache_files -d ${lmod_cache_dir} -t ${lmod_cache_timestamp_file} ${modpath}
+check_exit_code $? "Lmod cache updated" "Lmod cache update failed!"
+
+ls -lrt ${EASYBUILD_INSTALLPATH}/.lmod/cache

utils.sh

@@ -0,0 +1,28 @@
+function echo_green() {
+    echo -e "\e[32m$1\e[0m"
+}
+
+function echo_red() {
+    echo -e "\e[31m$1\e[0m"
+}
+
+function echo_yellow() {
+    echo -e "\e[33m$1\e[0m"
+}
+
+function fatal_error() {
+    echo_red "ERROR: $1" >&2
+    exit 1
+}
+
+function check_exit_code {
+    ec=$1
+    ok_msg=$2
+    fail_msg=$3
+
+    if [[ $ec -eq 0 ]]; then
+        echo_green "${ok_msg}"
+    else
+        fatal_error "${fail_msg}"
+    fi
+}