Skip to content

ci: extract CircleCI workflow routing into a script #21579

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ajsutton merged 1 commit into from
Jul 7, 2026
Merged

ci: extract CircleCI workflow routing into a script #21579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 6 additions & 133 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,146 +186,19 @@ jobs:
command: .circleci/scripts/collect-params.sh detect_all
environment:
c-only_docs_changes: "^docs/public-docs/"
# Decision tree: determines which workflows to enable based on trigger, branch, and params.
#
# Helpers (defined in workflow-helpers.sh):
# run foo → sets c-run_foo: true in JSON (maps 1:1 to
# "when: << pipeline.parameters.c-run_foo >>" in a continuation config)
# is_true x → checks if c-x is true in JSON (set by earlier steps)
# param x → reads raw value of c-x from JSON
# finalize → strips intermediate params, keeps only the listed whitelist + all c-run_* flags
#
# How to add a new workflow:
# 1. Add "c-run_your_workflow: {type: boolean, default: false}" in the relevant continuation config
# 2. Add "run your_workflow" in the appropriate branch below
# Decision tree: determines which workflows to enable based on trigger,
# branch, and params. The logic lives in compute-workflow-conditions.sh
# (using the run/is_true/param/finalize helpers from workflow-helpers.sh);
# each enabled c-run_<name> maps 1:1 to
# "when: << pipeline.parameters.c-run_<name> >>" in a continuation config.
- run:
name: Compute workflow conditions from pipeline parameters and store in JSON file
environment:
BRANCH: << pipeline.git.branch >>
TRIGGER_SOURCE: << pipeline.trigger_source >>
TAG: << pipeline.git.tag >>
SCHEDULE_NAME: << pipeline.schedule.name >>
command: |
set -euo pipefail
source .circleci/scripts/workflow-helpers.sh
init_json

case "${TRIGGER_SOURCE}" in

# Scheduled pipelines: map schedule name → workflows
scheduled_pipeline)
case "${SCHEDULE_NAME}" in
build_four_hours) run scheduled_todo_issues scheduled_cannon_full_tests ;;
build_daily) run scheduled_preimage_reproducibility scheduled_stale_check scheduled_heavy_fuzz_tests scheduled_daily_tests circleci_schedule_trigger_check ;;
esac
;;

# Webhook (push events)
webhook)
# --- Tag push ---
if [[ -n "${TAG}" ]]; then
run release

# =========================================================
# Three mutually exclusive lifecycle stages:
# =========================================================

# 1. PR (feature branch push)
# Runs on every push to a feature branch.
# Path-based gating: only changed areas are tested.
# Docs-only changes skip main/release entirely.
# ---------------------------------------------------------
elif [[ "${BRANCH}" != "develop" && ! "${BRANCH}" =~ ^gh-readonly-queue/ ]]; then
if is_true only_docs_changes; then
run ci_gate_skip
run contracts_feature_tests_short
run rust_ci_gate_short
run rust_e2e_gate_skip
else
run main
run release
if is_true contracts_changed; then
run contracts_feature_tests
else
run contracts_feature_tests_short
fi
if is_true rust_changes_detected; then
run rust_ci
run rust_e2e_ci
else
run rust_ci_gate_short
run rust_e2e_gate_skip
fi
if is_true circleci_changed; then
run circleci_schedule_trigger_check
fi
fi

# 2. Merge queue (pre-merge validation)
# Runs when GitHub merge queue picks up the PR.
# Full contract tests, path-gated rust/docs.
# ---------------------------------------------------------
elif [[ "${BRANCH}" =~ ^gh-readonly-queue/ ]]; then
run main
run release
run contracts_feature_tests
if is_true rust_changes_detected; then
run rust_ci
run rust_e2e_ci
else
run rust_ci_gate_short
run rust_e2e_gate_skip
fi
if is_true circleci_changed; then
run circleci_schedule_trigger_check
fi

# 3. After merge (develop push)
# Runs after the merge queue completes and pushes to develop.
# Adds expensive post-merge jobs: fault proofs, kontrol, prestate publishing.
# ---------------------------------------------------------
elif [[ "${BRANCH}" == "develop" ]]; then
run main
run release
run publish_contract_artifacts
run develop_fault_proofs
run develop_kontrol_tests
run contracts_feature_tests
if is_true rust_changes_detected; then
run rust_ci
run rust_e2e_ci
run kona_publish_prestates
else
run rust_ci_gate_short
run rust_e2e_gate_skip
fi
fi
;;

# API triggers: dispatch flags select workflows
api)
run release
if is_true main_dispatch && [[ "$(param github-event-type)" == "__not_set__" ]]; then
run main contracts_feature_tests
fi
if is_true fault_proofs_dispatch; then run develop_fault_proofs; fi
if is_true kontrol_dispatch; then run develop_kontrol_tests; fi
if is_true cannon_full_test_dispatch; then run scheduled_cannon_full_tests; fi
if is_true reproducibility_dispatch; then run scheduled_preimage_reproducibility; fi
if is_true stale_check_dispatch; then run scheduled_stale_check; fi
if is_true heavy_fuzz_dispatch; then run scheduled_heavy_fuzz_tests; fi
if is_true publish_contract_artifacts_dispatch; then run publish_contract_artifacts; fi
if is_true l2_fork_test_dispatch; then run l2_fork_test; fi
if is_true rust_ci_dispatch; then run rust_ci; fi
if is_true rust_e2e_dispatch; then run rust_e2e_ci; fi
if [[ "$(param github-event-type)" == "pull_request" && "$(param github-event-action)" == "labeled" ]]; then
run close_issue
fi
;;
esac

# Params to forward to continuation configs (everything else is stripped)
finalize "c-default_docker_image,c-rust_base_image,c-base_image,c-github-event-base64,c-go-cache-version,c-publish_contract_artifacts_ref"
command: .circleci/scripts/compute-workflow-conditions.sh
# Deep-merges the 3 continuation YAML configs (main, rust-ci, rust-e2e)
# into /tmp/merged-config.yml using yq. Later files win on key conflicts.
# explode(.) resolves YAML anchors/aliases so the output is self-contained.
Expand Down
142 changes: 142 additions & 0 deletions .circleci/scripts/compute-workflow-conditions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
#!/usr/bin/env bash
# Workflow routing decision tree for the CircleCI setup pipeline.
#
# Reads the change-detection / dispatch params already written to
# /tmp/pipeline-parameters.json by earlier steps and enables the matching
# c-run_* workflow flags based on the trigger source, branch, tag, and schedule.
#
# Inputs (set by the config.yml step environment):
# BRANCH, TRIGGER_SOURCE, TAG, SCHEDULE_NAME
#
# Helpers (workflow-helpers.sh):
# run foo → sets c-run_foo: true in JSON (maps 1:1 to
# "when: << pipeline.parameters.c-run_foo >>" in a continuation config)
# is_true x → checks if c-x is true in JSON (set by earlier steps)
# param x → reads raw value of c-x from JSON
# finalize → strips intermediate params, keeps only the listed whitelist + all c-run_* flags
#
# How to add a new workflow:
# 1. Add "c-run_your_workflow: {type: boolean, default: false}" in the relevant continuation config
# 2. Add "run your_workflow" in the appropriate branch below
set -euo pipefail

# shellcheck disable=SC1091 # sourced helper resolved at runtime, not by shellcheck
source "$(dirname "${BASH_SOURCE[0]}")/workflow-helpers.sh"
init_json

case "${TRIGGER_SOURCE}" in

# Scheduled pipelines: map schedule name → workflows
scheduled_pipeline)
case "${SCHEDULE_NAME}" in
build_four_hours) run scheduled_todo_issues scheduled_cannon_full_tests ;;
build_daily) run scheduled_preimage_reproducibility scheduled_stale_check scheduled_heavy_fuzz_tests scheduled_daily_tests circleci_schedule_trigger_check ;;
esac
;;

# Webhook (push events)
webhook)
# --- Tag push ---
if [[ -n "${TAG}" ]]; then
run release

# =========================================================
# Three mutually exclusive lifecycle stages:
# =========================================================

# 1. PR (feature branch push)
# Runs on every push to a feature branch.
# Path-based gating: only changed areas are tested.
# Docs-only changes skip main/release entirely.
# ---------------------------------------------------------
elif [[ "${BRANCH}" != "develop" && ! "${BRANCH}" =~ ^gh-readonly-queue/ ]]; then
if is_true only_docs_changes; then
run ci_gate_skip
run contracts_feature_tests_short
run rust_ci_gate_short
run rust_e2e_gate_skip
else
run main
run release
if is_true contracts_changed; then
run contracts_feature_tests
else
run contracts_feature_tests_short
fi
if is_true rust_changes_detected; then
run rust_ci
run rust_e2e_ci
else
run rust_ci_gate_short
run rust_e2e_gate_skip
fi
if is_true circleci_changed; then
run circleci_schedule_trigger_check
fi
fi

# 2. Merge queue (pre-merge validation)
# Runs when GitHub merge queue picks up the PR.
# Full contract tests, path-gated rust/docs.
# ---------------------------------------------------------
elif [[ "${BRANCH}" =~ ^gh-readonly-queue/ ]]; then
run main
run release
run contracts_feature_tests
if is_true rust_changes_detected; then
run rust_ci
run rust_e2e_ci
else
run rust_ci_gate_short
run rust_e2e_gate_skip
fi
if is_true circleci_changed; then
run circleci_schedule_trigger_check
fi

# 3. After merge (develop push)
# Runs after the merge queue completes and pushes to develop.
# Adds expensive post-merge jobs: fault proofs, kontrol, prestate publishing.
# ---------------------------------------------------------
elif [[ "${BRANCH}" == "develop" ]]; then
run main
run release
run publish_contract_artifacts
run develop_fault_proofs
run develop_kontrol_tests
run contracts_feature_tests
if is_true rust_changes_detected; then
run rust_ci
run rust_e2e_ci
run kona_publish_prestates
else
run rust_ci_gate_short
run rust_e2e_gate_skip
fi
fi
;;

# API triggers: dispatch flags select workflows
api)
run release
if is_true main_dispatch && [[ "$(param github-event-type)" == "__not_set__" ]]; then
run main contracts_feature_tests
fi
if is_true fault_proofs_dispatch; then run develop_fault_proofs; fi
if is_true kontrol_dispatch; then run develop_kontrol_tests; fi
if is_true cannon_full_test_dispatch; then run scheduled_cannon_full_tests; fi
if is_true reproducibility_dispatch; then run scheduled_preimage_reproducibility; fi
if is_true stale_check_dispatch; then run scheduled_stale_check; fi
if is_true heavy_fuzz_dispatch; then run scheduled_heavy_fuzz_tests; fi
if is_true publish_contract_artifacts_dispatch; then run publish_contract_artifacts; fi
if is_true l2_fork_test_dispatch; then run l2_fork_test; fi
if is_true rust_ci_dispatch; then run rust_ci; fi
if is_true rust_e2e_dispatch; then run rust_e2e_ci; fi
if [[ "$(param github-event-type)" == "pull_request" && "$(param github-event-action)" == "labeled" ]]; then
run close_issue
fi
;;
esac

# Params to forward to continuation configs (everything else is stripped)
finalize "c-default_docker_image,c-rust_base_image,c-base_image,c-github-event-base64,c-go-cache-version,c-publish_contract_artifacts_ref"
52 changes: 16 additions & 36 deletions .circleci/scripts/test-decision-tree.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
#!/usr/bin/env bash
# Dry-run test for the workflow decision tree in config.yml.
# Extracts the decision tree dynamically using yq, then runs it against
# a table of scenarios, asserting the expected c-run_* flags are set.
# Dry-run test for the workflow routing policy (compute-workflow-conditions.sh).
# Seeds the params JSON, sets the trigger/branch/tag/schedule environment, runs
# the routing script, then asserts the expected c-run_* flags are (or are not)
# set in the resulting JSON.
#
# Usage:
# bash .circleci/scripts/test-decision-tree.sh
Expand All @@ -10,28 +11,15 @@
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
CONFIG="${REPO_ROOT}/.circleci/config.yml"
OUTPUT="/tmp/pipeline-parameters.json"

# Extract the inline command from the "Compute workflow conditions..." step
STEP_NAME="Compute workflow conditions from pipeline parameters and store in JSON file"
DECISION_TREE=$(yq "
.jobs.prepare-continuation-config.steps[]
| select(.run.name == \"${STEP_NAME}\")
| .run.command
" "${CONFIG}")

if [[ -z "${DECISION_TREE}" ]]; then
echo "ERROR: Could not extract decision tree from ${CONFIG}" >&2
exit 1
fi
ROUTING_SCRIPT="${SCRIPT_DIR}/compute-workflow-conditions.sh"

# Strip the source/init_json/finalize lines — we handle those ourselves
DECISION_TREE=$(echo "${DECISION_TREE}" | grep -v '^\s*set -euo pipefail' \
| grep -v '^\s*source ' \
| grep -v '^\s*init_json' \
| grep -v '^\s*finalize ')
# Use an isolated params file so the test never writes the pipeline's real
# /tmp/pipeline-parameters.json. The routing script's finalize writes c-run_*
# flags, which the later collect-params/compute steps would otherwise inherit.
# Exported so the routing script (via workflow-helpers.sh) writes here too.
OUTPUT="$(mktemp)"
export OUTPUT
trap 'rm -f "${OUTPUT}"' EXIT

# --- Test harness ---
PASS=0
Expand Down Expand Up @@ -60,22 +48,14 @@ run_scenario() {
fi
done

# Seed the JSON
# Seed the JSON and run the routing policy as the pipeline would.
echo "${json_seed}" > "${OUTPUT}"
TRIGGER_SOURCE="${trigger}" BRANCH="${branch}" TAG="${tag}" SCHEDULE_NAME="${schedule}" \
bash "${ROUTING_SCRIPT}" >/dev/null || true

# Set environment
export TRIGGER_SOURCE="${trigger}"
export BRANCH="${branch}"
export TAG="${tag}"
export SCHEDULE_NAME="${schedule}"

# Source helpers and run decision tree
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/workflow-helpers.sh"
local _json
_json=$(cat "${OUTPUT}")

eval "${DECISION_TREE}" 2>/dev/null

# Check expected workflows are enabled
local all_pass=true
for wf in "${expected[@]}"; do
Expand Down
Loading
Loading