.gitlab-ci.yml

# Environment variables shared between scripts for all ORNL CI stages
.ornl_environment_template: &ornl_environment_variables
  variables:
    SCHEDULER_PARAMETERS: "-P CSC359 -nnodes 1 -W 30"
    WORKDIR: /gpfs/wolf/proj-shared/csc359/ci/${CI_PIPELINE_ID}
    
.ornl_test_script_template: &ornl_test_script_definition
  script:
    - |
      set -xv
      cd "$WORKDIR"
      MY_CLUSTER="ascent" ./BUILD.sh --test-only
      res=$?
      exit $res
  after_script:
    - |
      cd "$WORKDIR/.."
      rm -rf "$WORKDIR"

.ornl_script_template: &ornl_script_definition
  script:
    - |
      # Don't clean up this working directory - we need some of these files for
      # testing
      set -xv
      mkdir -p "$WORKDIR"
      cp -R ./* "$WORKDIR"
      cd "$WORKDIR"
      MY_CLUSTER="ascent" ./BUILD.sh --build-only || exit 1

.pnnl_script_template: &pnnl_script_definition
  script:
    - |
      #
      #  NOTES:  WORKDIR is on constance/marianas/newell
      #          ./      is only on the Kubernetes instance
      #
      set -xv
      export WORKDIR="$HOME/gitlab/$CI_JOB_ID/"
      mkdir -p "$WORKDIR"
      cp -R ./* "$WORKDIR"
      cd "$WORKDIR"
      touch output
      tail -f output &
      tailpid=$!

      if [[ $MY_CLUSTER == "marianas" ]]; then
        export SLURM_Q=`perl $WORKDIR/scripts/findIdleDLNodes.pl`
      fi

      sbatch -A EXASGD --exclusive -N 1 -n 8 -p $SLURM_Q -t $TIMELIMIT $SLURM_ARGS -o output -e output $WORKDIR/BUILD.sh $BUILD_SCRIPT_ARGS
      res=1
      set +xv
      while :;
        do
        if [[ "$(awk 'BEGIN{i=0}/BUILD_STATUS/{i++}END{print i}' output)" != "0" ]]; then
          kill $tailpid
          res=$(grep BUILD_STATUS output | tail -n 1 | cut -f2 -d':')
          break
        fi
        sleep 10
      done
      echo "finished batch job: $res"
      exit $res
  after_script:
    - |
      set -xv
      export WORKDIR="$HOME/gitlab/$CI_JOB_ID/"
      rm -rf "$WORKDIR"

.pnnl_tags_template: &pnnl_tags_definition
  tags:
    - k8s
    - ikp
    - exasgd
    - marianas

stages:
  - build
  - default_build
  - test

# For PNNL CI
build_on_marianas:
  stage: default_build
  variables:
    SLURM_Q: "dl"
    MY_CLUSTER: "marianas"
    TIMELIMIT: '1:30:00'
    SLURM_ARGS: --gres=gpu:1 --exclusive
  <<: *pnnl_tags_definition
  <<: *pnnl_script_definition
  rules:
    - if: '$CI_PROJECT_ROOT_NAMESPACE == "exasgd"'

build_on_newell:
  stage: default_build
  variables:
    SLURM_Q: "newell_shared"
    MY_CLUSTER: "newell"
    TIMELIMIT: '1:00:00'
    SLURM_ARGS: --gres=gpu:1 --exclusive
  <<: *pnnl_tags_definition
  <<: *pnnl_script_definition
  rules:
    - if: '$CI_PROJECT_ROOT_NAMESPACE == "exasgd"'

build_on_incline:
  allow_failure: true
  stage: default_build
  variables:
    SLURM_Q: incline
    MY_CLUSTER: incline
    TIMELIMIT: '1:00:00'
    SLURM_ARGS: --exclusive
    BUILD_SCRIPT_ARGS: --build-only
  <<: *pnnl_tags_definition
  <<: *pnnl_script_definition
  rules:
    - if: '$CI_PROJECT_ROOT_NAMESPACE == "exasgd"'

# For Ascent CI
build_on_login_node:
  stage: build
  tags:
    - nobatch
  rules:
    - if: '$CI_PROJECT_PATH == "ecpcitest/exasgd/hiop"'
  <<: *ornl_script_definition
  <<: *ornl_environment_variables

test_on_compute_node:
  stage: test
  dependencies:
    - build_on_login_node
  tags:
    - batch
  rules:
    - if: '$CI_PROJECT_PATH == "ecpcitest/exasgd/hiop"'
  <<: *ornl_test_script_definition
  <<: *ornl_environment_variables
# ---