[infra] publish rc to pypi as part of release process (#1449)

## Which issue does this PR close?

<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->

- Closes #1409.

## What changes are included in this PR?

Changes
- Trigger `publish` workflow only by matching specific tag names
(`v<major>.<minor>.<patch>` or
`v<major>.<minor>.<patch>-rc.<release_candidate>`) instead of all tags
- Add a step in `pypi` workflow to again validate the tag name 
- Edit `pypi` workflow to override the Cargo version for RC. This allows
us to upload to pypi as pre-release


<!--
Provide a summary of the modifications in this PR. List the main changes
such as new features, bug fixes, refactoring, or any other updates.
-->

## Are these changes tested?

<!--
Specify what test covers (unit test, integration test, etc.).

If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
-->
Yes, tested against personal fork.

Tested
- push invalid tag, i.e. `should-not-trigger`. ✅ does not trigger the
`publish` job
- push RC tag, i.e. `v0.6.0-rc.1`. ✅ triggers [`publish`
job](https://github.com/kevinjqliu/iceberg-rust/actions/runs/15815922351)
and [`pypi`
job](https://github.com/kevinjqliu/iceberg-rust/actions/runs/15815944806)
- push release tag, i.e. `v0.6.0`. ✅ triggers [`publish`
job](https://github.com/kevinjqliu/iceberg-rust/actions/runs/15815922462)
and [`pypi`
job](https://github.com/kevinjqliu/iceberg-rust/actions/runs/15815944240)

For both `v0.6.0-rc.1` and `v0.6.0`, verified the generated artifacts
(`wheels-sdist` and `wheels-ubuntu-latest-armv7l`) locally for the
correctly set python version number.

Author: kevinjqliu

.github/workflows/publish.yml

@@ -20,7 +20,10 @@ name: Publish
 on:
   push:
     tags:
-      - "*"
+      # Trigger this workflow when tag follows the versioning format: v<major>.<minor>.<patch> OR v<major>.<minor>.<patch>-rc.<release_candidate>
+      # Example valid tags: v0.4.0, v0.4.0-rc.1
+      - "v[0-9]+.[0-9]+.[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
   workflow_dispatch:
 
 env:

.github/workflows/release_python.yml

@@ -28,7 +28,7 @@ env:
   rust_msrv: "1.85"
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}-${{ github.event_name }}
   cancel-in-progress: true
 
 permissions:
@@ -42,11 +42,62 @@ jobs:
     steps:
       - run: echo 'The Publish workflow passed or was manually triggered'
 
-  sdist:
+  validate-release-tag:
     runs-on: ubuntu-latest
     needs: [check-cargo-publish]
+    outputs:
+      cargo-version: ${{ steps.validate.outputs.cargo-version }}
+      is-rc: ${{ steps.validate.outputs.is-rc }}
+    steps:
+      - name: Validate release tag format
+        id: validate
+        # Note, `workflow_run.head_branch` does not contain `refs/tags/` prefix, just the tag name, i.e. `v0.4.0` or `v0.4.0-rc.1`
+        # Valid formats: v<major>.<minor>.<patch> OR v<major>.<minor>.<patch>-rc.<release_candidate>
+        run: |
+          RELEASE_TAG="${{ github.event.workflow_run.head_branch }}"
+          echo "Validating release tag: $RELEASE_TAG"
+          if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then
+            echo "❌ Invalid release tag format: $RELEASE_TAG"
+            echo "Expected format: v<major>.<minor>.<patch> OR v<major>.<minor>.<patch>-rc.<release_candidate>"
+            echo "Examples: v0.4.0, v1.2.3-rc.1"
+            exit 1
+          fi
+          echo "✅ Release tag format is valid: $RELEASE_TAG"
+          
+          # Strip 'v' prefix for cargo version
+          CARGO_VERSION="${RELEASE_TAG#v}"
+          echo "Cargo version (without v prefix): $CARGO_VERSION"
+          
+          # Check if this is a release candidate
+          if [[ "$RELEASE_TAG" =~ -rc\.[0-9]+$ ]]; then
+            IS_RC="true"
+            echo "This is a release candidate"
+          else
+            IS_RC="false"
+            echo "This is a stable release"
+          fi
+          
+          # Set outputs for other jobs to use
+          echo "cargo-version=$CARGO_VERSION" >> $GITHUB_OUTPUT
+          echo "is-rc=$IS_RC" >> $GITHUB_OUTPUT
+
+  sdist:
+    runs-on: ubuntu-latest
+    needs: [validate-release-tag]
     steps:
       - uses: actions/checkout@v4
+
+      - name: Install cargo-edit
+        if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }}
+        run: cargo install cargo-edit
+      
+      - name: Set cargo version for RC
+        if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }}
+        working-directory: "bindings/python"
+        run: |
+          echo "Setting cargo version to: ${{ needs.validate-release-tag.outputs.cargo-version }}"
+          cargo set-version ${{ needs.validate-release-tag.outputs.cargo-version }}
+
       - uses: PyO3/maturin-action@v1
         with:
           working-directory: "bindings/python"
@@ -60,7 +111,7 @@ jobs:
 
   wheels:
     runs-on: "${{ matrix.os }}"
-    needs: [check-cargo-publish]
+    needs: [validate-release-tag]
     strategy:
       matrix:
         include:
@@ -75,6 +126,18 @@ jobs:
           - { os: ubuntu-latest, target: "armv7l" }
     steps:
       - uses: actions/checkout@v4
+
+      - name: Install cargo-edit
+        if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }}
+        run: cargo install cargo-edit
+      
+      - name: Set cargo version for RC
+        if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }}
+        working-directory: "bindings/python"
+        run: |
+          echo "Setting cargo version to: ${{ needs.validate-release-tag.outputs.cargo-version }}"
+          cargo set-version ${{ needs.validate-release-tag.outputs.cargo-version }}
+
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
@@ -99,9 +162,6 @@ jobs:
     name: Publish Python 🐍 distribution 📦 to Pypi
     needs: [sdist, wheels]
     runs-on: ubuntu-latest
-    # Only publish to PyPi if the tag is not a pre-release OR if manually triggered
-    # Note, `workflow_run.head_branch` does not contain `refs/tags/` prefix, just the tag name, i.e. `v0.4.0` or `v0.4.0-rc.1`
-    if: ${{ !contains(github.event.workflow_run.head_branch, '-') || github.event_name == 'workflow_dispatch' }}
 
     environment:
       name: pypi