Don't hardcode path to singularity

Adds ability to configure what command and path
to use for calling singularity
also adds some sanity checks to see that the set command
is sensible and exists

Author: Nortamo

configs/local.yaml

@@ -33,9 +33,18 @@ force:
     conda_arch: 'Linux-x86_64'
     max_num_cpus: 4
     # No logic currently to refer to other variables here
-    # So either implement that, throw them to $TMPDIR
-    #
+    # So either implement that, throw them to $TMPDIR 
     workdir: $PWD   
+    # What executable name or path to use for singularity
+    # For fully dynamic behavior 
+    # You can just set singularity or apptainer here.
+    # Keep in mind that this could break installations
+    # if e.g the user installs singularity via conda
+    # when user namespaces are not enabled 
+    # Environment variables can be used here 
+    # but they will be expanded during build,
+    # not during runtime
+    singularity_executable_path: "singularity"
 
 # Value will override any user set value
 

configs/lumi.yaml

@@ -41,6 +41,16 @@ force:
     # So either implement that, throw them to $TMPDIR
     #
     workdir: $PWD 
+    # What executable name or path to use for singularity
+    # For fully dynamic behavior 
+    # You can just set singularity or apptainer here.
+    # Keep in mind that this could break installations
+    # if e.g the user installs singularity via conda
+    # when user namespaces are not enabled 
+    # Environment variables can be used here 
+    # but they will be expanded during build,
+    # not during runtime
+    singularity_executable_path: "/usr/bin/singularity"
 
 # Value will override any user set value
 

configs/mahti.yaml

@@ -39,6 +39,16 @@ force:
     # So either implement that, throw them to $TMPDIR
     #
     workdir: $PWD   
+    # What executable name or path to use for singularity
+    # For fully dynamic behavior 
+    # You can just set singularity or apptainer here.
+    # Keep in mind that this could break installations
+    # if e.g the user installs singularity via conda
+    # when user namespaces are not enabled 
+    # Environment variables can be used here 
+    # but they will be expanded during build,
+    # not during runtime
+    singularity_executable_path: "/usr/bin/singularity"
 
 # Value will override any user set value
 

configs/puhti.yaml

@@ -40,6 +40,16 @@ force:
     # So either implement that, throw them to $TMPDIR
     #
     workdir: $PWD   
+    # What executable name or path to use for singularity
+    # For fully dynamic behavior 
+    # You can just set singularity or apptainer here.
+    # Keep in mind that this could break installations
+    # if e.g the user installs singularity via conda
+    # when user namespaces are not enabled 
+    # Environment variables can be used here 
+    # but they will be expanded during build,
+    # not during runtime
+    singularity_executable_path: "/usr/bin/singularity"
 
 # Value will override any user set value
 

create_inst.sh

@@ -54,18 +54,18 @@ export install_root=$CW_INSTALLATION_PATH
 
 
 if [[ "$CW_UPDATE_INSTALLATION" == "yes" ]];then
-    _CONTAINER_EXEC="singularity --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_SOURCE_MOUNT_POINT:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_SOURCE_MOUNT_POINT:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
     export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH,$_inst_path/_bin:$_inst_path/bin"
     print_info "Copying installation to writable area, might take a while" 1
     print_info "$(readlink -f $CW_INSTALLATION_PREFIX)" 1
     $_CONTAINER_EXEC cp -a $CW_SOURCE_MOUNT_POINT/. $CW_INSTALLATION_PATH || { print_err "Failed to copy some files, most likely incorrect file permissions inside the squashfs image" && false ; }
 elif [[ "$CW_MODE" == "wrapdisk" ]];then
     export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH,$CW_WRAP_SRC:$CW_SOURCE_MOUNT_POINT"
-    _CONTAINER_EXEC="singularity --silent exec _deploy/$CW_CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE"
 
 else
     export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH"
-    _CONTAINER_EXEC="singularity --silent exec _deploy/$CW_CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE"
 fi
 cp ./_sing_inst_script.sh _pre_install.sh _post_install.sh _inst_dir 
 print_info "Running installation script" 1

frontends/containerize

@@ -98,6 +98,19 @@ else
    export _inst_path=$(realpath -s $PWD/$CW_INSTALLATION_PREFIX )
 fi
 
+if ! command -v $CW_SINGULARITY_EXECUTABLE_PATH &>/dev/null; then
+	{ print_err "The command $CW_SINGULARITY_EXECUTABLE_PATH does not exists
+	Verify that the value for 'singularity_executable_path' in $CW_GLOBAL_YAML is correct"; false ;}
+fi
+
+print_info "Validating the set singularity command $CW_SINGULARITY_EXECUTABLE_PATH" 2
+num_res=$($CW_SINGULARITY_EXECUTABLE_PATH --help | grep "build\|exec\|shell" 2>/dev/null | wc -l)
+if [[ "$num_res" -lt 3 ]];then
+	{ print_err "$CW_SINGULARITY_EXECUTABLE_PATH does not seem to be a valid apptainer/singularity executable"; false ;}
+fi
+
+
+
 $M_SCRIPT_DIR/../pre.sh || { print_err "Failed getting container image"; false ;}
 # Nothing to do if we are just wrapping a container
 if [[ ! "$CW_MODE" == "wrapcont" ]];then

generate_wrappers.sh

@@ -15,14 +15,14 @@ echo "#!/bin/bash" > _deploy/common.sh
 
 
 if [[ "$CW_MODE" == "wrapcont" ]];then
-    _CONTAINER_EXEC="/usr/bin/singularity --silent exec  _deploy/$CW_CONTAINER_IMAGE"
-    _RUN_CMD="/usr/bin/singularity --silent exec \$DIR/../\$CONTAINER_IMAGE"
-    _SHELL_CMD="/usr/bin/singularity --silent shell \$DIR/../\$CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec  _deploy/$CW_CONTAINER_IMAGE"
+    _RUN_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec \$DIR/../\$CONTAINER_IMAGE"
+    _SHELL_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent shell \$DIR/../\$CONTAINER_IMAGE"
 else
-    _CONTAINER_EXEC="/usr/bin/singularity --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_INSTALLATION_PATH:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_INSTALLATION_PATH:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
     echo "SQFS_IMAGE=$CW_SQFS_IMAGE" >> _deploy/common.sh
-    _RUN_CMD="/usr/bin/singularity --silent exec  \$DIR/../\$CONTAINER_IMAGE"
-    _SHELL_CMD="/usr/bin/singularity --silent shell \$DIR/../\$CONTAINER_IMAGE"
+    _RUN_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec  \$DIR/../\$CONTAINER_IMAGE"
+    _SHELL_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent shell \$DIR/../\$CONTAINER_IMAGE"
 fi
 
 # Need to unset the path, otherwise we might be stuck in a nasty loop

pre.sh

@@ -14,8 +14,11 @@ mkdir -p  $CW_BUILD_TMPDIR/_deploy/
 
 if [[ ! -e $CW_CONTAINER_SRC ]]; then 
     print_info "Fetching container $CW_CONTAINER_SRC"  1
-    singularity --silent pull $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE $CW_CONTAINER_SRC || \
+    $CW_SINGULARITY_EXECUTABLE_PATH --silent pull $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE $CW_CONTAINER_SRC || \
     { print_err "Failed fetching container"; exit 1 ;}
+    if [[ ! -e $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE ]];then
+	{ print_err "$CW_SINGULARITY_EXECUTABLE_PATH pull returned success, but no container was fetched"; exit 1 ;}
+    fi
 else
     if [[ ${CW_SHARE_CONTAINER+defined} && ${CW_SHARE_CONTAINER} == "yes" ]];then
         print_info "Using container $CW_CONTAINER_SRC" 1

tests/test-03.sh

@@ -32,6 +32,19 @@ rule ghelp:
         """
 ' > Snakefile
 
+cat ../../default_config/config.yaml | sed  "s@singularity_executable_path.*@singularity_executable_path: 'ThisIsNotACommand'@g" > my_config.yaml 
+export CW_GLOBAL_YAML=$( readlink -f my_config.yaml)
+t_run "conda-containerize new --mamba env2.yml --prefix S | grep 'ThisIsNotACommand does not exists'" "Exit if configured singularity command does not exist" 
+cat ../../default_config/config.yaml | sed  "s@singularity_executable_path.*@singularity_executable_path: 'my_sing_command'@g" > my_config.yaml 
+echo "#!/bin/bash" > my_sing_command
+echo "exit 0" >> my_sing_command
+chmod +x my_sing_command
+export PATH=$PATH:$PWD
+t_run "conda-containerize new --mamba env2.yml --prefix S | grep 'does not seem to be a valid apptainer/singularity executable'" "Exit if configured singularity command seems broken" 
+# Run the rest of the test with singularity found from path.
+cat ../../default_config/config.yaml | sed  "s@singularity_executable_path.*@singularity_executable_path: 'singularity'@g" > my_config.yaml 
+
+
 t_run "conda-containerize new --mamba env2.yml --prefix S" "mamba works" 
 t_run "conda-containerize new --mamba env.yml --prefix Gdal" "gdal installed"
 export OPATH=$PATH