Skip to content

Commit

Permalink
chore(ci): introduce daily pre-releases (#4535)
Browse files Browse the repository at this point in the history 
* fix(parameters): make cache aware of single vs multiple calls

Signed-off-by: heitorlessa <[email protected]>

* chore: cleanup, add test for single and nested

Signed-off-by: heitorlessa <[email protected]>

* chore(ci): add first centralized reusable workflow

* chore: remove playground

* chore: remove staged file

Signed-off-by: heitorlessa <[email protected]>

* chore(ci): initial pre-release workflow

* chore(ci): adjust workflow name to pre-release

* chore(ci): install poetry beforehand

* chore(ci): check source code beforehand

* chore(ci): stage provenance file as part of release PR

* fix(ci): stage entire source code as composite actions don't allow vars input

* chore: fix syntax for download provenance

* chore: move changelog to 10am UTC to allow prerelease PR merge

* chore: ensure provenance is staged in git

* chore: remove fork guard

* chore: use new restricted pre-release env; restore fork guard

* chore: trigger workflow on weekdays only

* docs(homepage): reorganize install to fit alpha releases

* docs(homepage): add alpha releases section

* chore: typo in pip command

Co-authored-by: Leandro Damascena <[email protected]>
Signed-off-by: Heitor Lessa <[email protected]>

* fix(ci): remove duplicate version bumping

* docs(maintainers): improve release notes section based on feedback

Signed-off-by: heitorlessa <[email protected]>

* docs(maintainers): add note about alpha releases

Signed-off-by: heitorlessa <[email protected]>

* docs: mention business days (weekdays); leandro's feedback

* docs: add edge cases note in versioning validation

---------

Signed-off-by: heitorlessa <[email protected]>
Signed-off-by: Heitor Lessa <[email protected]>
Co-authored-by: Leandro Damascena <[email protected]>
  • Loading branch information
@heitorlessa @leandrodamascena
heitorlessa and leandrodamascena authored Jun 20, 2024
1 parent c2d8f89 commit 6782b6c
Show file tree
Hide file tree
Showing 5 changed files with 442 additions and 129 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/build_changelog.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ on:
# branches:
# - develop
schedule:
# Note: run daily at 7am UTC time until upstream git-chlog uses stable sorting
- cron: "0 7 * * *"
# Note: run daily at 10am UTC time until upstream git-chlog uses stable sorting
- cron: "0 10 * * *"

permissions:
contents: read
Expand Down
275 changes: 275 additions & 0 deletions .github/workflows/pre-release.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,275 @@
name: Pre-Release

# PRE-RELEASE PROCESS
#
# === Automated activities ===
#
# 1. [Seal] Bump to release version and export source code with integrity hash
# 2. [Quality check] Restore sealed source code, run tests, linting, security and complexity base line
# 3. [Build] Restore sealed source code, create and export hashed build artifact for PyPi release (wheel, tarball)
# 4. [Provenance] Generates provenance for build, signs attestation with GitHub OIDC claims to confirm it came from this release pipeline, commit, org, repo, branch, hash, etc.
# 5. [Release] Restore built artifact, and publish package to PyPi prod repository
# 6. [PR to bump version] Restore sealed source code, and create a PR to update trunk with latest released project metadata

# NOTE
#
# See MAINTAINERS.md "Releasing a new version" for release mechanisms
#
# Every job is isolated and starts a new fresh container.

env:
RELEASE_COMMIT: ${{ github.sha }}

on:
workflow_dispatch:
inputs:
skip_code_quality:
description: "Skip tests, linting, and baseline. Only use if release fail for reasons beyond our control and you need a quick release."
default: false
type: boolean
required: false
skip_pypi:
description: "Skip publishing to PyPi. Used for testing release steps."
default: false
type: boolean
required: false
schedule:
# Note: run daily on weekdays at 8am UTC time
- cron: "0 8 * * 1-5"

permissions:
contents: read

jobs:

# This job bumps the package version to the pre-release version
# creates an integrity hash from the source code
# uploads the artifact with the integrity hash as the key name
# so subsequent jobs can restore from a trusted point in time to prevent tampering
seal:
# ignore forks
if: github.repository == 'aws-powertools/powertools-lambda-python'

runs-on: ubuntu-latest
permissions:
contents: read
outputs:
integrity_hash: ${{ steps.seal_source_code.outputs.integrity_hash }}
artifact_name: ${{ steps.seal_source_code.outputs.artifact_name }}
RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }}
steps:
# NOTE: Different from prod release, we need both poetry and source code available in earlier steps to bump and verify.

# We use a pinned version of Poetry to be certain it won't modify source code before we create a hash
- name: Install poetry
run: |
pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0
pipx inject poetry git+https://github.com/monim67/poetry-bumpversion@315fe3324a699fa12ec20e202eb7375d4327d1c4 # v0.3.1
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ env.RELEASE_COMMIT }}

- name: Bump and export release version
id: release_version
run: |
RELEASE_VERSION="$(poetry version prerelease --short | head -n1 | tr -d '\n')"
echo "RELEASE_VERSION=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT"
- name: Verifies pre-release version semantics
# verify pre-release semantics before proceeding to avoid versioning pollution
# e.g., 2.40.0a1 and 2.40.0b2 are valid while 2.40.0 is not
# NOTE. we do it in a separate step to handle edge cases like
# `poetry` CLI uses immutable install, versioning behaviour could change even in a minor version (we had breaking changes before)
# a separate step allows us to pinpoint what happened (before/after)
run: |
if [[ ! "$RELEASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+[a-b].*$ ]]; then
echo "Version $VERSION doesn't look like a pre-release version; aborting"
exit 1
fi
env:
RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION}}

- name: Seal and upload
id: seal_source_code
uses: ./.github/actions/seal
with:
artifact_name_prefix: "source"

# This job runs our automated test suite, complexity and security baselines
# it ensures previously merged have been tested as part of the pull request process
#
# NOTE
#
# we don't upload the artifact after testing to prevent any tampering of our source code dependencies
quality_check:
needs: seal
runs-on: ubuntu-latest
permissions:
contents: read
steps:
# NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev)
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code
uses: ./.github/actions/seal-restore
with:
integrity_hash: ${{ needs.seal.outputs.integrity_hash }}
artifact_name: ${{ needs.seal.outputs.artifact_name }}

- name: Debug cache restore
run: cat pyproject.toml

- name: Install poetry
run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0
- name: Set up Python
uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
with:
python-version: "3.12"
cache: "poetry"
- name: Install dependencies
run: make dev
- name: Run all tests, linting and baselines
run: make pr

# This job creates a release artifact (tar.gz, wheel)
# it checks out code from release commit for custom actions to work
# then restores the sealed source code (overwrites any potential tampering)
# it's done separately from release job to enforce least privilege.
# We export just the final build artifact for release
build:
runs-on: ubuntu-latest
needs: [quality_check, seal]
permissions:
contents: read
outputs:
integrity_hash: ${{ steps.seal_build.outputs.integrity_hash }}
artifact_name: ${{ steps.seal_build.outputs.artifact_name }}
attestation_hashes: ${{ steps.encoded_hash.outputs.attestation_hashes }}
steps:
# NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev)
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code
uses: ./.github/actions/seal-restore
with:
integrity_hash: ${{ needs.seal.outputs.integrity_hash }}
artifact_name: ${{ needs.seal.outputs.artifact_name }}

- name: Install poetry
run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0
- name: Set up Python
uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
with:
python-version: "3.12"
cache: "poetry"

- name: Build python package and wheel
run: poetry build

- name: Seal and upload
id: seal_build
uses: ./.github/actions/seal
with:
artifact_name_prefix: "build"
files: "dist/"

# NOTE: SLSA retraces our build to its artifact to ensure it wasn't tampered
# coupled with GitHub OIDC, SLSA can then confidently sign it came from this release pipeline+commit+branch+org+repo+actor+integrity hash
- name: Create attestation encoded hash for provenance
id: encoded_hash
working-directory: dist
run: echo "attestation_hashes=$(sha256sum ./* | base64 -w0)" >> "$GITHUB_OUTPUT"

# This job creates a provenance file that describes how our release was built (all steps)
# after it verifies our build is reproducible within the same pipeline
# it confirms that its own software and the CI build haven't been tampered with (Trust but verify)
# lastly, it creates and sign an attestation (multiple.intoto.jsonl) that confirms
# this build artifact came from this GitHub org, branch, actor, commit ID, inputs that triggered this pipeline, and matches its integrity hash
# NOTE: supply chain threats review (we protect against all of them now): https://slsa.dev/spec/v1.0/threats-overview
provenance:
needs: [seal, build]
permissions:
contents: write # nested job explicitly require despite upload assets being set to false
actions: read # To read the workflow path.
id-token: write # To sign the provenance.
# NOTE: provenance fails if we use action pinning... it's a Github limitation
# because SLSA needs to trace & attest it came from a given branch; pinning doesn't expose that information
# https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md#referencing-the-slsa-generator
uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]
with:
base64-subjects: ${{ needs.build.outputs.attestation_hashes }}
upload-assets: false # we upload its attestation in create_tag job, otherwise it creates a new release

# This job uses release artifact to publish to PyPi
# it exchanges JWT tokens with GitHub to obtain PyPi credentials
# since it's already registered as a Trusted Publisher.
# It uses the sealed build artifact (.whl, .tar.gz) to release it
release:
needs: [build, seal, provenance]
environment: pre-release
runs-on: ubuntu-latest
permissions:
id-token: write # OIDC for PyPi Trusted Publisher feature
env:
RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }}
steps:
# NOTE: we need actions/checkout in order to use our local actions (e.g., ./.github/actions)
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code
uses: ./.github/actions/seal-restore
with:
integrity_hash: ${{ needs.build.outputs.integrity_hash }}
artifact_name: ${{ needs.build.outputs.artifact_name }}

- name: Upload to PyPi prod
if: ${{ !inputs.skip_pypi }}
uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14

# Creates a PR with the latest version we've just released
# since our trunk is protected against any direct pushes from automation
bump_version:
needs: [release, seal, provenance]
permissions:
contents: write # create-pr action creates a temporary branch
pull-requests: write # create-pr action creates a PR using the temporary branch
runs-on: ubuntu-latest
steps:
# NOTE: we need actions/checkout to authenticate and configure git first
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code
uses: ./.github/actions/seal-restore
with:
integrity_hash: ${{ needs.seal.outputs.integrity_hash }}
artifact_name: ${{ needs.seal.outputs.artifact_name }}

- name: Download provenance
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: ${{needs.provenance.outputs.provenance-name}}

- name: Update provenance
run: mkdir -p "${PROVENANCE_DIR}" && mv "${PROVENANCE_FILE}" "${PROVENANCE_DIR}/"
env:
PROVENANCE_FILE: ${{ needs.provenance.outputs.provenance-name }}
PROVENANCE_DIR: provenance/${{ needs.seal.outputs.RELEASE_VERSION}}

- name: Create PR
id: create-pr
uses: ./.github/actions/create-pr
with:
files: "pyproject.toml aws_lambda_powertools/shared/version.py provenance/"
temp_branch_prefix: "ci-bump"
pull_request_title: "chore(ci): new pre-release ${{ needs.seal.outputs.RELEASE_VERSION }}"
github_token: ${{ secrets.GITHUB_TOKEN }}
Loading

0 comments on commit 6782b6c

Please sign in to comment.