Prepare Main CD Python release

Author: recmanj

.github/workflows/conda-index-lock-self-hosted.yml

@@ -32,34 +32,87 @@ concurrency:
 jobs:
   index:
     name: Index Conda Repository
-    runs-on: ubuntu-latest
+    runs-on: platform-builder-Debian-12
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5
 
-      - name: Setup Conda
-        uses: conda-incubator/setup-miniconda@v3
-        with:
-          auto-activate-base: true
-          activate-environment: ""
+      - name: Setup conda env dir
+        shell: bash -el {0}
+        run: |
+          echo "CONDA_ENVS_DIRS=$RUNNER_TEMP/envs" >> "$GITHUB_ENV"
+          mkdir -p "$RUNNER_TEMP/envs"
+
+      - name: Create conda environment
+        shell: bash -el {0}
+        run: |
+          INDEX_ENV="conda-index-lock-${GITHUB_RUN_ID}"
+          echo "INDEX_ENV=$INDEX_ENV" >> "$GITHUB_ENV"
+          mamba env create -n "$INDEX_ENV"
 
       - name: Install dependencies
         shell: bash -el {0}
         run: |
-          conda install -y conda-index requests
+          eval "$(conda shell.bash hook)"
+          conda activate "$INDEX_ENV"
+          mamba install -y conda-index requests
 
       - name: Download artifacts from caller workflow
+        shell: bash -el {0}
         env:
           GH_TOKEN: ${{ secrets.GH_REPO_READ_TOKEN }}
+          REPO: ${{ inputs.caller_repo }}
+          RUN_ID: ${{ inputs.caller_run_id }}
+          ARTIFACT_NAME: ${{ inputs.package_artifact_name }}
         run: |
-          echo "Downloading artifact '${{ inputs.package_artifact_name }}' from run ${{ inputs.caller_run_id }}"
-          echo "Caller repository: ${{ inputs.caller_repo }}"
+          eval "$(conda shell.bash hook)"
+          conda activate "$INDEX_ENV"
+
+          echo "Downloading artifact '$ARTIFACT_NAME' from run $RUN_ID in $REPO"
+
+          python3 << 'EOF'
+          import os
+          import sys
+          import zipfile
+          import requests
 
-          # Download artifact from the calling workflow
-          gh run download ${{ inputs.caller_run_id }} \
-            --repo ${{ inputs.caller_repo }} \
-            --name ${{ inputs.package_artifact_name }} \
-            --dir ./packages
+          repo = os.environ["REPO"]
+          run_id = os.environ["RUN_ID"]
+          artifact_name = os.environ["ARTIFACT_NAME"]
+          token = os.environ["GH_TOKEN"]
+
+          # Get artifact list
+          url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/artifacts"
+          headers = {"Authorization": f"Bearer {token}"}
+          resp = requests.get(url, headers=headers)
+          resp.raise_for_status()
+
+          # Find artifact ID
+          artifact_id = None
+          for artifact in resp.json()["artifacts"]:
+              if artifact["name"] == artifact_name:
+                  artifact_id = artifact["id"]
+                  break
+
+          if not artifact_id:
+              print(f"Artifact '{artifact_name}' not found", file=sys.stderr)
+              sys.exit(1)
+
+          # Download artifact zip
+          url = f"https://api.github.com/repos/{repo}/actions/artifacts/{artifact_id}/zip"
+          resp = requests.get(url, headers=headers)
+          resp.raise_for_status()
+
+          # Extract to ./packages
+          os.makedirs("./packages", exist_ok=True)
+          with open("/tmp/artifact.zip", "wb") as f:
+              f.write(resp.content)
+          with zipfile.ZipFile("/tmp/artifact.zip") as z:
+              z.extractall("./packages")
+          os.remove("/tmp/artifact.zip")
+
+          print("Downloaded and extracted artifact")
+          EOF
 
           echo "Downloaded packages:"
           find ./packages -name "*.tar.bz2" | sort
@@ -70,6 +123,9 @@ jobs:
           NEXUS_URL: ${{ inputs.nexus_url }}
           NEXUS_TOKEN: ${{ inputs.nexus_token }}
         run: |
+          eval "$(conda shell.bash hook)"
+          conda activate "$INDEX_ENV"
+
           echo "Indexing packages to: $NEXUS_URL"
 
           python "${{ github.workspace }}/cd-actions/python-conda/scripts/rebuild_conda_index.py" \

.github/workflows/conda-index-lock.yml

@@ -140,7 +140,7 @@ jobs:
 
       - name: Build with Python Conda
         if: ${{ matrix.type == 'python-conda' }}
-        uses: ecmwf/reusable-workflows/cd-actions/python-conda@cd-actions
+        uses: ecmwf/reusable-workflows/cd-actions/python-conda@v2
         with:
           name: ${{ matrix.name }}
           config: ${{ matrix.config }}
@@ -153,7 +153,7 @@ jobs:
 
       - name: Build with Python PyPI
         if: ${{ matrix.type == 'python-pypi' }}
-        uses: ecmwf/reusable-workflows/cd-actions/python-pypi@cd-actions
+        uses: ecmwf/reusable-workflows/cd-actions/python-pypi@v2
         with:
           name: ${{ matrix.name }}
           config: ${{ matrix.config }}
@@ -188,7 +188,7 @@ jobs:
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Create GitHub release
-        uses: ecmwf/reusable-workflows/cd-actions/release@cd-actions
+        uses: ecmwf/reusable-workflows/cd-actions/release@v2
         with:
           config: ${{ steps.prepare-config.outputs.config }}
           ref_name: ${{ inputs.ref_name }}

.github/workflows/main-cd.yml

@@ -168,83 +168,6 @@ runs:
             find "$output_folder" -type f | head -20
             exit 1
         fi
-    - name: Upload artifacts
-      if: ${{ inputs.dry_run != 'true' }}
-      shell: bash -el {0}
-      run: |
-        nexus_url="${{ steps.config.outputs.nexus_url }}"
-        nexus_token="${{ steps.config.outputs.nexus_token }}"
-        artifact_name="${{ github.run_id }}-${{ inputs.name }}"
-
-        echo "Acquiring conda index lock and uploading to Nexus: $nexus_url"
-        echo "Artifact: $artifact_name"
-        echo ""
-
-        # Run the lock acquisition script
-        python3 "${{ github.action_path }}/scripts/acquire_lock.py" \
-            --nexus-url "$nexus_url" \
-            --nexus-token "$nexus_token" \
-            --artifact-name "$artifact_name" \
-            --caller-run-id "${{ github.run_id }}" \
-            --caller-repo "${{ github.repository }}" \
-            --gh-pat "${{ inputs.gh_pat }}"
-
-        echo ""
-        echo "All packages and indexes uploaded to Nexus successfully!"
-
-    - name: Test package installation from Nexus
-      if: ${{ inputs.dry_run != 'true' && inputs.skip_installation_test != 'true' }}
-      shell: bash -el {0}
-      run: |
-        output_folder="${{ steps.config.outputs.output_folder }}"
-        nexus_url="${{ steps.config.outputs.nexus_url }}"
-
-        echo "Testing package installation from Nexus..."
-
-        # Get package name and version from built packages
-        packages=$(find "$output_folder" -name "*.tar.bz2" -type f)
-
-        for package in $packages; do
-            package_name=$(basename "$package")
-            echo ""
-            echo "Testing installation of: $package_name"
-
-            # Extract package name without version/build info
-            pkg_base=$(echo "$package_name" | sed 's/-.*//')
-
-            # Create a test environment
-            test_env="test-install-${GITHUB_RUN_ID}-${RANDOM}"
-            echo "Creating test environment: $test_env"
-
-            eval "$(conda shell.bash hook)"
-
-            # Create environment and install package from Nexus
-            if mamba create -n "$test_env" -c "$nexus_url" -c conda-forge "$pkg_base" -y; then
-                echo "✓ Successfully installed $pkg_base from Nexus"
-
-                # Verify package is installed
-                conda activate "$test_env"
-                if conda list | grep -q "$pkg_base"; then
-                    echo "✓ Package $pkg_base verified in environment"
-                else
-                    echo "✗ Package $pkg_base not found in environment"
-                    conda deactivate
-                    conda env remove -n "$test_env" -y
-                    exit 1
-                fi
-                conda deactivate
-
-                # Clean up test environment
-                conda env remove -n "$test_env" -y
-                echo "Test environment removed"
-            else
-                echo "✗ Failed to install $pkg_base from Nexus"
-                exit 1
-            fi
-        done
-
-        echo ""
-        echo "All packages successfully installed from Nexus!"
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v4

cd-actions/python-conda/action.yml

@@ -11,7 +11,7 @@
 
 # Configuration
 LOCK_REPO = "ecmwf/reusable-workflows"
-LOCK_WORKFLOW = "conda-index-lock-self-hosted.yml"
+LOCK_WORKFLOW = "conda-index-lock.yml"
 MAX_WAIT = 1800  # 30 minutes
 POLL_INTERVAL = 10  # seconds
 GITHUB_API = "https://api.github.com"
@@ -68,7 +68,7 @@ def main():
         f"/repos/{LOCK_REPO}/actions/workflows/{LOCK_WORKFLOW}/dispatches",
         method="POST",
         data={
-            "ref": "cd-actions",
+            "ref": "v2",
             "inputs": {
                 "nexus_url": args.nexus_url,
                 "nexus_token": args.nexus_token,