From 9b6c7e84c30990a3c152d6b3a8b03091fde37b8c Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Mon, 23 Sep 2024 15:50:37 +0200 Subject: [PATCH 1/9] use miniforge --- .github/workflows/ci.yml | 56 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43b7a82..4b72e80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,11 +22,11 @@ jobs: with: submodules: recursive - name: Setup env + # CONDA_ROOT="$(realpath $GITHUB_WORKSPACE/..)/install/miniconda3" + # echo "CONDA_ROOT=$CONDA_ROOT" >> "$GITHUB_ENV" run: | INSTALLED_DIR="$(realpath $GITHUB_WORKSPACE)/third_party/install" echo "INSTALLED_DIR=$INSTALLED_DIR" >> "$GITHUB_ENV" - CONDA_ROOT="$(realpath $GITHUB_WORKSPACE/..)/install/miniconda3" - echo "CONDA_ROOT=$CONDA_ROOT" >> "$GITHUB_ENV" echo 'IMEX_SHA<> $GITHUB_ENV cat imex_version.txt >> $GITHUB_ENV echo 'EOF' >> $GITHUB_ENV @@ -36,27 +36,31 @@ jobs: with: path: third_party/install/** key: ${{ runner.os }}-build-imex-${{ env.IMEX_CACHE_NUMBER }}-${{ env.IMEX_SHA }} - - name: Miniconda - run: | - env - rm -rf $CONDA_ROOT - cd $GITHUB_WORKSPACE/.. - rm -f Miniconda3-*.sh - # CPKG=Miniconda3-latest-Linux-x86_64.sh - CPKG=Miniconda3-py311_24.3.0-0-Linux-x86_64.sh - wget -q https://repo.anaconda.com/miniconda/$CPKG - bash $CPKG -u -b -f -p $CONDA_ROOT - export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" - conda install -c conda-forge --override-channels python git-lfs 'conda-build<=24.1.2' - # aahhhh bug in conda somewhere - sed -i "s,\#\!/usr/bin/env python,#\!$CONDA_ROOT/bin/python," $CONDA_ROOT/*bin/conda - conda clean --all -y + - name: Miniforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + # environment-file: etc/example-environment.yml + # - name: Miniconda + # run: | + # env + # rm -rf $CONDA_ROOT + # cd $GITHUB_WORKSPACE/.. + # rm -f Miniconda3-*.sh + # CPKG=Miniforge3-Linux-x86_64.sh + # wget -q https://github.com/conda-forge/miniforge/releases/latest/download/$CPKG + # bash $CPKG -u -b -f -p $CONDA_ROOT + # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} + # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" + # conda install -c conda-forge --override-channels python git-lfs 'conda-build<=24.1.2' + # # aahhhh bug in conda somewhere + # sed -i "s,\#\!/usr/bin/env python,#\!$CONDA_ROOT/bin/python," $CONDA_ROOT/*bin/conda + # conda clean --all -y - name: Build SHARPY + # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} + # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | env - export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" cd conda-recipe conda build --numpy 1.26 -c conda-forge --override-channels --no-anaconda-upload --no-test . - name: Save IMEX cache @@ -66,15 +70,15 @@ jobs: path: third_party/install/** key: ${{ runner.os }}-build-imex-${{ env.IMEX_CACHE_NUMBER }}-${{ env.IMEX_SHA }} - name: Install SHARPY + # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} + # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | - export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" conda create --use-local -c conda-forge --override-channels -n sharpy -v sharpy pytest pytest-timeout conda activate sharpy - name: Test SHARPY (pytest) + # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} + # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | - export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" conda activate sharpy cd test pytest . -v -s --timeout=30 @@ -84,9 +88,9 @@ jobs: mpirun -n 4 pytest . -v -s --timeout=30 cd - - name: Run examples + # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} + # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | - export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" conda activate sharpy cd examples SHARPY_FALLBACK=numpy python -u ./stencil-2d.py 5 1024 star 4 From 2cb3279459fde2aa80fec3c3eb2ca393c304d0ba Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Mon, 23 Sep 2024 15:57:15 +0200 Subject: [PATCH 2/9] install conda build --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b72e80..69fdc91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | env + conda install -y conda-build cd conda-recipe conda build --numpy 1.26 -c conda-forge --override-channels --no-anaconda-upload --no-test . - name: Save IMEX cache From e946ac2816b05fb0b9aca1840856c19d2b3f6bbd Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Tue, 24 Sep 2024 09:56:54 +0200 Subject: [PATCH 3/9] conda init --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69fdc91..8456e39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,7 @@ jobs: # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | + conda init bash conda create --use-local -c conda-forge --override-channels -n sharpy -v sharpy pytest pytest-timeout conda activate sharpy - name: Test SHARPY (pytest) From 0e96ef7c0760e813f9fafb07cced698786c70a42 Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Tue, 24 Sep 2024 10:07:45 +0200 Subject: [PATCH 4/9] test --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8456e39..bc122af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: pull_request: defaults: run: - shell: bash + shell: bash -l {0} env: IMEX_CACHE_NUMBER: 4 # Increase to reset cache jobs: @@ -41,6 +41,13 @@ jobs: with: miniforge-version: latest # environment-file: etc/example-environment.yml + - name: Test Conda + run: | + conda create -n testenv conda-build + conda activate testenv + which python + conda deactivate + conda env remove -n testenv # - name: Miniconda # run: | # env @@ -74,7 +81,6 @@ jobs: # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | - conda init bash conda create --use-local -c conda-forge --override-channels -n sharpy -v sharpy pytest pytest-timeout conda activate sharpy - name: Test SHARPY (pytest) From 155fe044b4b215f3751f77838d63caa5c0636db7 Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Tue, 24 Sep 2024 10:24:12 +0200 Subject: [PATCH 5/9] cleanup --- .github/workflows/ci.yml | 42 +++++++--------------------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc122af..85f09aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,16 +1,20 @@ name: Sharpy CI run-name: ${{ github.actor }} CI for sharpy permissions: read-all + on: push: branches: - main pull_request: + defaults: run: shell: bash -l {0} + env: IMEX_CACHE_NUMBER: 4 # Increase to reset cache + jobs: build_and_test: runs-on: ubuntu-22.04 @@ -22,8 +26,6 @@ jobs: with: submodules: recursive - name: Setup env - # CONDA_ROOT="$(realpath $GITHUB_WORKSPACE/..)/install/miniconda3" - # echo "CONDA_ROOT=$CONDA_ROOT" >> "$GITHUB_ENV" run: | INSTALLED_DIR="$(realpath $GITHUB_WORKSPACE)/third_party/install" echo "INSTALLED_DIR=$INSTALLED_DIR" >> "$GITHUB_ENV" @@ -36,41 +38,17 @@ jobs: with: path: third_party/install/** key: ${{ runner.os }}-build-imex-${{ env.IMEX_CACHE_NUMBER }}-${{ env.IMEX_SHA }} - - name: Miniforge + - name: Setup Miniforge uses: conda-incubator/setup-miniconda@v3 with: miniforge-version: latest # environment-file: etc/example-environment.yml - - name: Test Conda - run: | - conda create -n testenv conda-build - conda activate testenv - which python - conda deactivate - conda env remove -n testenv - # - name: Miniconda - # run: | - # env - # rm -rf $CONDA_ROOT - # cd $GITHUB_WORKSPACE/.. - # rm -f Miniconda3-*.sh - # CPKG=Miniforge3-Linux-x86_64.sh - # wget -q https://github.com/conda-forge/miniforge/releases/latest/download/$CPKG - # bash $CPKG -u -b -f -p $CONDA_ROOT - # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" - # conda install -c conda-forge --override-channels python git-lfs 'conda-build<=24.1.2' - # # aahhhh bug in conda somewhere - # sed -i "s,\#\!/usr/bin/env python,#\!$CONDA_ROOT/bin/python," $CONDA_ROOT/*bin/conda - # conda clean --all -y - name: Build SHARPY - # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | env conda install -y conda-build cd conda-recipe - conda build --numpy 1.26 -c conda-forge --override-channels --no-anaconda-upload --no-test . + conda build --numpy 1.26 --no-anaconda-upload --no-test . - name: Save IMEX cache uses: actions/cache/save@v3 if: steps.cache-imex.outputs.cache-hit != 'true' @@ -78,14 +56,10 @@ jobs: path: third_party/install/** key: ${{ runner.os }}-build-imex-${{ env.IMEX_CACHE_NUMBER }}-${{ env.IMEX_SHA }} - name: Install SHARPY - # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | - conda create --use-local -c conda-forge --override-channels -n sharpy -v sharpy pytest pytest-timeout + conda create --use-local -n sharpy -v sharpy pytest pytest-timeout conda activate sharpy - name: Test SHARPY (pytest) - # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | conda activate sharpy cd test @@ -96,8 +70,6 @@ jobs: mpirun -n 4 pytest . -v -s --timeout=30 cd - - name: Run examples - # export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH} - # eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)" run: | conda activate sharpy cd examples From fb6394a599df3ff0a0483ea7b9ec1ae8cca91d9b Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Tue, 24 Sep 2024 10:37:25 +0200 Subject: [PATCH 6/9] -c file:///user/fschlimb/mambaforge/envs/imex-dev/conda-bld/ --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85f09aa..c7f632e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: key: ${{ runner.os }}-build-imex-${{ env.IMEX_CACHE_NUMBER }}-${{ env.IMEX_SHA }} - name: Install SHARPY run: | - conda create --use-local -n sharpy -v sharpy pytest pytest-timeout + conda create --use-local -c file://${{ env.CONDA_PREFIX }}/conda-bld/ -n sharpy -v sharpy pytest pytest-timeout conda activate sharpy - name: Test SHARPY (pytest) run: | From c3e8c743e415706622d776f9b4f80adff711d96c Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Tue, 24 Sep 2024 10:46:51 +0200 Subject: [PATCH 7/9] tests --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7f632e..ccc7f0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,11 @@ jobs: with: miniforge-version: latest # environment-file: etc/example-environment.yml + - name: TestTest + run: | + echo CONDA_PREFIX=${CONDA_PREFIX} + echo CONDA_PREFIX=${{CONDA_PREFIX}} + echo CONDA_PREFIX=${{env.CONDA_PREFIX}} - name: Build SHARPY run: | env From 1985e9c62971cbbded41d569dc509ec33f86292e Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Tue, 24 Sep 2024 10:47:51 +0200 Subject: [PATCH 8/9] tests --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccc7f0d..8c12011 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,6 @@ jobs: - name: TestTest run: | echo CONDA_PREFIX=${CONDA_PREFIX} - echo CONDA_PREFIX=${{CONDA_PREFIX}} echo CONDA_PREFIX=${{env.CONDA_PREFIX}} - name: Build SHARPY run: | From fc3cf8831372215d1f322759b515afda43d8bdcb Mon Sep 17 00:00:00 2001 From: "Schlimbach, Frank" Date: Tue, 24 Sep 2024 10:49:35 +0200 Subject: [PATCH 9/9] test --- .github/workflows/ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c12011..8340641 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,10 +43,6 @@ jobs: with: miniforge-version: latest # environment-file: etc/example-environment.yml - - name: TestTest - run: | - echo CONDA_PREFIX=${CONDA_PREFIX} - echo CONDA_PREFIX=${{env.CONDA_PREFIX}} - name: Build SHARPY run: | env @@ -61,7 +57,7 @@ jobs: key: ${{ runner.os }}-build-imex-${{ env.IMEX_CACHE_NUMBER }}-${{ env.IMEX_SHA }} - name: Install SHARPY run: | - conda create --use-local -c file://${{ env.CONDA_PREFIX }}/conda-bld/ -n sharpy -v sharpy pytest pytest-timeout + conda create --use-local -c file://${CONDA_PREFIX}/conda-bld/ -n sharpy -v sharpy pytest pytest-timeout conda activate sharpy - name: Test SHARPY (pytest) run: |