get_node_memory_size

partition=$1
walltime=$2

percentage=90
scheduler=`cat $WORKDIR/scheduler`
version=`cat $WORKDIR/version`
job_submit=`cat $WORKDIR/job_submit`
job_cancel=`cat $WORKDIR/job_cancel`

NULL=0

excluded_nodes=`cat $WORKDIR/excluded_${partition}_nodes`

echo "Trying to determine the available memory per ${partition} node. This may last up to ${walltime} seconds, do not interrupt."

echo "#! /bin/bash"                           > $WORKDIR/get_size_of_memory
echo "#SBATCH --job-name=memory"             >> $WORKDIR/get_size_of_memory
echo "#SBATCH --output=$WORKDIR/memory.out"  >> $WORKDIR/get_size_of_memory
echo "#SBATCH --error=$WORKDIR/memory.err"   >> $WORKDIR/get_size_of_memory
echo "#SBATCH --time=00:00:"$walltime        >> $WORKDIR/get_size_of_memory
echo "#SBATCH --cpus-per-task=1"             >> $WORKDIR/get_size_of_memory

if  [ ${partition} == gpu ] ; then
    echo "#SBATCH --gres=gpu:0"              >> $WORKDIR/get_size_of_memory
fi

if  [[ $partition == gpu ]] && [[ -e /etc/slurm/gres.conf ]]
then
    echo "#SBATCH --gres=gpu:0"              >> $WORKDIR/get_size_of_memory
fi

echo "#SBATCH --mem=2gb"                     >> $WORKDIR/get_size_of_memory
echo "#SBATCH --partition="$partition        >> $WORKDIR/get_size_of_memory
echo "#SBATCH --exclude=${excluded_nodes}"   >> $WORKDIR/get_size_of_memory
echo "WORKDIR=$WORKDIR"                      >> $WORKDIR/get_size_of_memory
echo "free -h"                               >> $WORKDIR/get_size_of_memory
chmod u+x                                       $WORKDIR/get_size_of_memory

rm -f $WORKDIR/memory.* tmp2

$job_submit -Q $WORKDIR/get_size_of_memory

timeout ${walltime}s $vini_dir/wait_until_jobs_finish

if [ -e $WORKDIR/memory.out ]
then
    line=`head -2  $WORKDIR/memory.out | tail -1`
    memsize=`echo $line | awk '{print $2}'`
    suffix=gb
    factor=1024
    echo $memsize > tmp
    memsize=$(tr -dc '0-9' <<< $memsize) #deleting non-numeric chars
    grep G tmp > tmp2                    #now check what we have (GB or TB)
    nochars=`wc -l < tmp2`
    if [ $nochars -ne $NULL ]
    then                                 #we have Gigabytes
        memsize=$(( memsize*${percentage}/100 ))
        memsize=${memsize}${suffix}
    else                                 #we have Terabytes. Multiply with 1024 
        memsize=`echo ${memsize} $factor | awk '{print $1 * $2}'`
        memsize=$(( memsize*${percentage}/100 ))
        memsize=${memsize}${suffix}
    fi
    echo "Found $memsize available memory per ${partition} node."
    echo $memsize > $WORKDIR/${partition}_memsize
else
    ${job_cancel} -u $USER ; echo
    echo "Failed to access any ${partition} node. A probable reason is that all nodes are either busy or down."
    echo "You may consider at least 90% of the total installed memory as available."
    read -p "Write the available memory size (in GB) per ${partition} node here:" memsize
    echo $memsize > $WORKDIR/${partition}_memsize
fi