Merge pull request #20 from CSCfi/no_hardcoded

Don't hardcode path to singularity

Author: Nortamo

configs/local.yaml

@@ -33,9 +33,18 @@ force:
     conda_arch: 'Linux-x86_64'
     max_num_cpus: 4
     # No logic currently to refer to other variables here
-    # So either implement that, throw them to $TMPDIR
-    #
+    # So either implement that, throw them to $TMPDIR 
     workdir: $PWD   
+    # What executable name or path to use for singularity
+    # For fully dynamic behavior 
+    # You can just set singularity or apptainer here.
+    # Keep in mind that this could break installations
+    # if e.g the user installs singularity via conda
+    # when user namespaces are not enabled 
+    # Environment variables can be used here 
+    # but they will be expanded during build,
+    # not during runtime
+    singularity_executable_path: "singularity"
 
 # Value will override any user set value
 

configs/lumi.yaml

@@ -41,6 +41,16 @@ force:
     # So either implement that, throw them to $TMPDIR
     #
     workdir: $PWD 
+    # What executable name or path to use for singularity
+    # For fully dynamic behavior 
+    # You can just set singularity or apptainer here.
+    # Keep in mind that this could break installations
+    # if e.g the user installs singularity via conda
+    # when user namespaces are not enabled 
+    # Environment variables can be used here 
+    # but they will be expanded during build,
+    # not during runtime
+    singularity_executable_path: "/usr/bin/singularity"
 
 # Value will override any user set value
 

configs/mahti.yaml

@@ -39,6 +39,16 @@ force:
     # So either implement that, throw them to $TMPDIR
     #
     workdir: $PWD   
+    # What executable name or path to use for singularity
+    # For fully dynamic behavior 
+    # You can just set singularity or apptainer here.
+    # Keep in mind that this could break installations
+    # if e.g the user installs singularity via conda
+    # when user namespaces are not enabled 
+    # Environment variables can be used here 
+    # but they will be expanded during build,
+    # not during runtime
+    singularity_executable_path: "/usr/bin/singularity"
 
 # Value will override any user set value
 

configs/puhti.yaml

@@ -40,6 +40,16 @@ force:
     # So either implement that, throw them to $TMPDIR
     #
     workdir: $PWD   
+    # What executable name or path to use for singularity
+    # For fully dynamic behavior 
+    # You can just set singularity or apptainer here.
+    # Keep in mind that this could break installations
+    # if e.g the user installs singularity via conda
+    # when user namespaces are not enabled 
+    # Environment variables can be used here 
+    # but they will be expanded during build,
+    # not during runtime
+    singularity_executable_path: "/usr/bin/singularity"
 
 # Value will override any user set value
 

create_inst.sh

@@ -54,18 +54,18 @@ export install_root=$CW_INSTALLATION_PATH
 
 
 if [[ "$CW_UPDATE_INSTALLATION" == "yes" ]];then
-    _CONTAINER_EXEC="singularity --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_SOURCE_MOUNT_POINT:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_SOURCE_MOUNT_POINT:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
     export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH,$_inst_path/_bin:$_inst_path/bin"
     print_info "Copying installation to writable area, might take a while" 1
     print_info "$(readlink -f $CW_INSTALLATION_PREFIX)" 1
     $_CONTAINER_EXEC cp -a $CW_SOURCE_MOUNT_POINT/. $CW_INSTALLATION_PATH || { print_err "Failed to copy some files, most likely incorrect file permissions inside the squashfs image" && false ; }
 elif [[ "$CW_MODE" == "wrapdisk" ]];then
     export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH,$CW_WRAP_SRC:$CW_SOURCE_MOUNT_POINT"
-    _CONTAINER_EXEC="singularity --silent exec _deploy/$CW_CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE"
 
 else
     export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH"
-    _CONTAINER_EXEC="singularity --silent exec _deploy/$CW_CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE"
 fi
 cp ./_sing_inst_script.sh _pre_install.sh _post_install.sh _inst_dir 
 print_info "Running installation script" 1

frontends/containerize

@@ -98,6 +98,19 @@ else
    export _inst_path=$(realpath -s $PWD/$CW_INSTALLATION_PREFIX )
 fi
 
+if ! command -v $CW_SINGULARITY_EXECUTABLE_PATH &>/dev/null; then
+	{ print_err "The command $CW_SINGULARITY_EXECUTABLE_PATH does not exists
+	Verify that the value for 'singularity_executable_path' in $CW_GLOBAL_YAML is correct"; false ;}
+fi
+
+print_info "Validating the set singularity command $CW_SINGULARITY_EXECUTABLE_PATH" 2
+num_res=$($CW_SINGULARITY_EXECUTABLE_PATH --help | grep "build\|exec\|shell" 2>/dev/null | wc -l)
+if [[ "$num_res" -lt 3 ]];then
+	{ print_err "$CW_SINGULARITY_EXECUTABLE_PATH does not seem to be a valid apptainer/singularity executable"; false ;}
+fi
+
+
+
 $M_SCRIPT_DIR/../pre.sh || { print_err "Failed getting container image"; false ;}
 # Nothing to do if we are just wrapping a container
 if [[ ! "$CW_MODE" == "wrapcont" ]];then

generate_wrappers.sh

@@ -15,14 +15,14 @@ echo "#!/bin/bash" > _deploy/common.sh
 
 
 if [[ "$CW_MODE" == "wrapcont" ]];then
-    _CONTAINER_EXEC="/usr/bin/singularity --silent exec  _deploy/$CW_CONTAINER_IMAGE"
-    _RUN_CMD="/usr/bin/singularity --silent exec \$DIR/../\$CONTAINER_IMAGE"
-    _SHELL_CMD="/usr/bin/singularity --silent shell \$DIR/../\$CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec  _deploy/$CW_CONTAINER_IMAGE"
+    _RUN_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec \$DIR/../\$CONTAINER_IMAGE"
+    _SHELL_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent shell \$DIR/../\$CONTAINER_IMAGE"
 else
-    _CONTAINER_EXEC="/usr/bin/singularity --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_INSTALLATION_PATH:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
+    _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_INSTALLATION_PATH:image-src=/ _deploy/$CW_CONTAINER_IMAGE"
     echo "SQFS_IMAGE=$CW_SQFS_IMAGE" >> _deploy/common.sh
-    _RUN_CMD="/usr/bin/singularity --silent exec  \$DIR/../\$CONTAINER_IMAGE"
-    _SHELL_CMD="/usr/bin/singularity --silent shell \$DIR/../\$CONTAINER_IMAGE"
+    _RUN_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec  \$DIR/../\$CONTAINER_IMAGE"
+    _SHELL_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent shell \$DIR/../\$CONTAINER_IMAGE"
 fi
 
 # Need to unset the path, otherwise we might be stuck in a nasty loop

pre.sh

@@ -14,8 +14,11 @@ mkdir -p  $CW_BUILD_TMPDIR/_deploy/
 
 if [[ ! -e $CW_CONTAINER_SRC ]]; then 
     print_info "Fetching container $CW_CONTAINER_SRC"  1
-    singularity --silent pull $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE $CW_CONTAINER_SRC || \
+    $CW_SINGULARITY_EXECUTABLE_PATH --silent pull $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE $CW_CONTAINER_SRC || \
     { print_err "Failed fetching container"; exit 1 ;}
+    if [[ ! -e $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE ]];then
+	{ print_err "$CW_SINGULARITY_EXECUTABLE_PATH pull returned success, but no container was fetched"; exit 1 ;}
+    fi
 else
     if [[ ${CW_SHARE_CONTAINER+defined} && ${CW_SHARE_CONTAINER} == "yes" ]];then
         print_info "Using container $CW_CONTAINER_SRC" 1

tests/test-03.sh

@@ -32,6 +32,19 @@ rule ghelp:
         """
 ' > Snakefile
 
+cat ../../default_config/config.yaml | sed  "s@singularity_executable_path.*@singularity_executable_path: 'ThisIsNotACommand'@g" > my_config.yaml 
+export CW_GLOBAL_YAML=$( readlink -f my_config.yaml)
+t_run "conda-containerize new --mamba env2.yml --prefix S | grep 'ThisIsNotACommand does not exists'" "Exit if configured singularity command does not exist" 
+cat ../../default_config/config.yaml | sed  "s@singularity_executable_path.*@singularity_executable_path: 'my_sing_command'@g" > my_config.yaml 
+echo "#!/bin/bash" > my_sing_command
+echo "exit 0" >> my_sing_command
+chmod +x my_sing_command
+export PATH=$PATH:$PWD
+t_run "conda-containerize new --mamba env2.yml --prefix S | grep 'does not seem to be a valid apptainer/singularity executable'" "Exit if configured singularity command seems broken" 
+# Run the rest of the test with singularity found from path.
+cat ../../default_config/config.yaml | sed  "s@singularity_executable_path.*@singularity_executable_path: 'singularity'@g" > my_config.yaml 
+
+
 t_run "conda-containerize new --mamba env2.yml --prefix S" "mamba works" 
 t_run "conda-containerize new --mamba env.yml --prefix Gdal" "gdal installed"
 export OPATH=$PATH