hpc-io · jeanbez · Jan 10, 2025 · Jan 9, 2025
@@ -3,7 +3,7 @@
 set -eu -o pipefail
 
 sudo apt-get update
-sudo apt-get install libopenmpi-dev libhdf5-dev
+sudo apt-get install libopenmpi-dev libhdf5-dev uuid-dev
 
 # libfabric
 wget https://github.com/ofiwg/libfabric/archive/refs/tags/v1.12.1.tar.gz

@@ -1,7 +1,7 @@
 name: NERSC
 
 on:
-  pull_request_target
+  pull_request
 
 env:
   PR_NUMBER: ${{ github.event.number }}
@@ -33,6 +33,6 @@ jobs:
         run: |
           PR_SHA=$(git rev-parse --short "$GITHUB_SHA")
           git fetch origin pull/${PR_NUMBER}/head:PR-${PR_SHA}
-          git remote add gitlab https://${{ secrets.NERSC_GITLAB_TOKEN_NAME }}:${{ secrets.NERSC_GITLAB_TOKEN }}@${{ secrets.NERSC_GITLAB_URL }}
+          git remote add gitlab https://${{ secrets.GITLAB_TOKEN_NAME }}:${{ secrets.GITLAB_TOKEN }}@${{ secrets.GITLAB_URL }}
           git checkout PR-${PR_SHA}
           git push -f gitlab -u PR-${PR_SHA}
@@ -1,20 +1,54 @@
 variables:
     PDC_BUILD_PATH: "${CI_PROJECT_DIR}/build"
     PDC_INSTALL_PATH: "${CI_PROJECT_DIR}/install"
-
+    GIT_CLONE_PATH: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}"
+
 stages:
     - build
     - test
     - metrics
 
+.report-status:
+    variables:
+        STATUS_PROJECT: "hpc-io/pdc"
+        STATUS_NAME: "NERSC / Perlmutter"
+    script:
+        # For complete details on the GitHub API please see:
+        # https://developer.github.com/v3/repos/statuses
+        - |
+          curl -L \
+          -X POST \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}" \
+          -d "{\"state\":\"${CI_JOB_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"context\":\"${STATUS_NAME}\"}"
+
+pending:
+    stage: .pre
+    extends:
+        - .report-status
+
+success:
+    stage: .post
+    extends:
+        - .report-status
+
+failure:
+    stage: .post
+    extends:
+        - .report-status
+    rules:
+        - when: on_failure
+
 perlmutter-no-cache-build:
     stage: build
     rules:
     - if: '$METRICS == null'
       when: manual
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com
     allow_failure: false
     tags:
         - perlmutter
@@ -42,7 +76,7 @@ perlmutter-cache-build:
       when: manual
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     allow_failure: false
     tags:
         - perlmutter
@@ -72,7 +106,7 @@ perlmutter-no-cache-parallel-pdc:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -83,8 +117,8 @@ perlmutter-no-cache-parallel-pdc:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-pdc"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-pdc"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-pdc"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-pdc"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/no-cache
@@ -95,7 +129,7 @@ perlmutter-no-cache-parallel-obj:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -107,8 +141,8 @@ perlmutter-no-cache-parallel-obj:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-obj"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-obj"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-obj"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-obj"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/no-cache
@@ -119,7 +153,7 @@ perlmutter-no-cache-parallel-cont:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -131,8 +165,8 @@ perlmutter-no-cache-parallel-cont:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-cont"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-cont"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-cont"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-cont"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/no-cache
@@ -143,7 +177,7 @@ perlmutter-no-cache-parallel-prop:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -155,8 +189,8 @@ perlmutter-no-cache-parallel-prop:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-prop"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-prop"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-prop"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-prop"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/no-cache
@@ -167,7 +201,7 @@ perlmutter-no-cache-parallel-region:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -179,8 +213,8 @@ perlmutter-no-cache-parallel-region:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-region"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-region"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-region"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-region"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/no-cache
@@ -191,7 +225,7 @@ perlmutter-no-cache-parallel-region-all:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -203,8 +237,8 @@ perlmutter-no-cache-parallel-region-all:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-region-all"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-region-all"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-region-all"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-region-all"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/no-cache
@@ -220,7 +254,7 @@ perlmutter-cache-parallel-pdc:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -231,8 +265,8 @@ perlmutter-cache-parallel-pdc:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-pdc"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-pdc"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-pdc"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-pdc"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/cache
@@ -243,7 +277,7 @@ perlmutter-cache-parallel-obj:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -255,8 +289,8 @@ perlmutter-cache-parallel-obj:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-obj"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-obj"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-obj"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-obj"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/cache
@@ -267,7 +301,7 @@ perlmutter-cache-parallel-cont:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -279,8 +313,8 @@ perlmutter-cache-parallel-cont:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-cont"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-cont"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-cont"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-cont"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/cache
@@ -291,7 +325,7 @@ perlmutter-cache-parallel-prop:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -303,8 +337,8 @@ perlmutter-cache-parallel-prop:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-prop"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-prop"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-prop"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-prop"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/cache
@@ -315,7 +349,7 @@ perlmutter-cache-parallel-region:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -327,8 +361,8 @@ perlmutter-cache-parallel-region:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-region"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-region"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-region"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-region"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/cache
@@ -339,7 +373,7 @@ perlmutter-cache-parallel-region-all:
     stage: test
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -351,8 +385,8 @@ perlmutter-cache-parallel-region-all:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-region-all"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-region-all"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-region-all"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-region-all"
     script:        
         - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
         - cd ${PDC_BUILD_PATH}/perlmutter/cache
@@ -363,7 +397,7 @@ perlmutter-metrics-build:
     stage: build
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
         - if: '$METRICS == "true"'
     allow_failure: false
@@ -390,7 +424,7 @@ perlmutter-metrics:
     stage: metrics
     id_tokens:
         SITE_ID_TOKEN:
-            aud: https://software.nersc.gov/
+            aud: https://gitlab.com/
     rules:
         - if: '$METRICS == "true"'
     needs:
@@ -401,8 +435,8 @@ perlmutter-metrics:
         SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=${PDC_QUEUE} --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
-        PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-metrics"
-        PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-metrics"
+        PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/pdc-tmp-metrics"
+        PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/pdc-data-metrics"
         PDC_CLIENT_LOOKUP: "NONE"
         PDC_SERVER: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/pdc_server.exe"
         PDC_SERVER_CLOSE: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/close_server"