Merge branch '2023.06-software.eessi.io' of github.com:EESSI/software-layer into riscv

Author: bedroge

Diff for: .github/workflows/scorecards.yml

@@ -19,7 +19,7 @@ jobs:
   analysis:
     if: github.repository_owner == 'EESSI'   # Prevent running on forks
     name: Scorecards analysis
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       # Needed to upload the results to code-scanning dashboard.
       security-events: write

Diff for: .github/workflows/test-software.eessi.io.yml

@@ -15,7 +15,7 @@ env:
       - nvidia/cc80
 jobs:
   check_missing:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -59,20 +59,6 @@ jobs:
               # first check the CPU-only builds for this CPU target
               echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)"
               for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
-                  if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then
-                      if grep -q 2022b <<<"${easystack_file}"; then
-                        # skip the check of installed software on zen4 for foss/2022b builds
-                        continue
-                      fi
-                      if [[ $easystack_file == *"rebuilds"* ]]; then
-                        # Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that
-                        new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml)
-                        # first clean out the options then clean out the .eb name
-                        sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack
-                        diff --unified=0 "$easystack_file" "$new_easystack" || :
-                        easystack_file="$new_easystack"
-                      fi
-                  fi
                   echo "check missing installations for ${easystack_file}..."
                   ./check_missing_installations.sh ${easystack_file}
                   ec=$?

Diff for: .github/workflows/test_eessi_container_script.yml

@@ -9,7 +9,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   eessi_container_script:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:

Diff for: .github/workflows/test_licenses.yml

@@ -8,7 +8,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

Diff for: .github/workflows/tests.yml

@@ -8,10 +8,10 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python: [3.7, 3.8, 3.9, '3.10']
+        python: [3.8, 3.9, '3.10']
       fail-fast: false
     steps:
     - name: checkout

Diff for: .github/workflows/tests_archdetect.yml

@@ -8,7 +8,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         proc_cpuinfo:
@@ -18,6 +18,7 @@ jobs:
           - x86_64/amd/zen3/Azure-CentOS7-7V73X
           - x86_64/amd/zen4/Azure-Alma8-9V33X
           - x86_64/amd/zen4/Shinx-RHEL8-9654
+          - aarch64/a64fx/Deucalion-Rocky85
           - aarch64/neoverse_n1/Azure-Ubuntu20-Altra
           - aarch64/neoverse_n1/AWS-awslinux-graviton2
           - aarch64/neoverse_v1/AWS-awslinux-graviton3

Diff for: .github/workflows/tests_archdetect_nvidia_gpu.yml

@@ -7,7 +7,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         fake_nvidia_smi_script:

Diff for: .github/workflows/tests_eessi_module.yml

@@ -8,7 +8,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   basic_checks:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -63,7 +63,7 @@ jobs:
           set -e  # Re-enable exit on non-zero status
 
   lmod_and_init_script_comparison:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -136,7 +136,7 @@ jobs:
           fi
 
   make_sure_load_and_unload_work:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -204,4 +204,4 @@ jobs:
               echo "Test for checking env variables FAILED"  >&2
               diff --unified=0 "${initial_env_file}" "${module_cycled_file}" 
               exit 1
-          fi
+          fi

Diff for: .github/workflows/tests_init.yml

@@ -8,10 +8,10 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python: [3.7, 3.8, 3.9, '3.10']
+        python: [3.8, 3.9, '3.10']
       fail-fast: false
     steps:
     - name: checkout

Diff for: .github/workflows/tests_init_module.yml

@@ -9,7 +9,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:

Diff for: .github/workflows/tests_readme.yml

@@ -14,7 +14,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

Diff for: .github/workflows/tests_scripts.yml

@@ -28,7 +28,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - name: checkout
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

Diff for: EESSI-install-software.sh

@@ -135,8 +135,9 @@ else
       # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set
       source $TOPDIR/init/minimal_eessi_env
 
-      # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect)
-      mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+      # make sure the the software and modules directory exist
+      # (since it's expected by init/eessi_environment_variables when using archdetect and by the EESSI module)
+      mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
   )
 fi
 
@@ -340,7 +341,15 @@ else
         if [ -f ${easystack_file} ]; then
             echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
 
-            ${EB} --easystack ${easystack_file} --robot
+            if [[ ${easystack_file} == *"/rebuilds/"* ]]; then
+                # the removal script should have removed the original directory and created a new and empty one
+                # to work around permission issues:
+                # https://github.com/EESSI/software-layer/issues/556
+                echo_yellow "This is a rebuild, so using --try-amend=keeppreviousinstall=True to reuse the already created directory"
+                ${EB} --easystack ${easystack_file} --robot --try-amend=keeppreviousinstall=True
+            else
+                ${EB} --easystack ${easystack_file} --robot
+            fi
             ec=$?
 
             # copy EasyBuild log file if EasyBuild exited with an error

Diff for: EESSI-remove-software.sh

@@ -125,13 +125,18 @@ if [ $EUID -eq 0 ]; then
                     # Two dirname invocations, so returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2
                     app_installprefix=$(dirname $(dirname ${app_modulepath}))
                     app_dir=${app_installprefix}/software/${app}
+                    app_subdirs=$(find ${app_dir} -mindepth 1 -maxdepth 1 -type d)
                     app_module=${app_installprefix}/modules/all/${app}.lua
                     echo_yellow "Removing ${app_dir} and ${app_module}..."
                     rm -rf ${app_dir}
                     rm -rf ${app_module}
-                    # recreate some directory to work around permission denied
-                    # issues when rebuilding the package
-                    mkdir -p ${app_dir}/easybuild
+                    # recreate the installation directory and do an ls on the first-level subdirectories to work around
+                    # permission issues when reinstalling the application (see https://github.com/EESSI/software-layer/issues/556)
+                    echo_yellow "Recreating an empty ${app_dir}..."
+                    mkdir -p ${app_dir}
+                    # these subdirs don't (and shouldn't) exist, but we need to do the ls anyway as a workaround,
+                    # so redirect to /dev/null and ignore the exit code
+                    ls ${app_subdirs} >& /dev/null || true
                 done
             else
                 fatal_error "Easystack file ${easystack_file} not found!"