From bfafc82ae81c1dff66f1b5bf51067b4eeb50a649 Mon Sep 17 00:00:00 2001 From: Andrea Sorbini Date: Wed, 17 Apr 2024 23:36:09 -0700 Subject: [PATCH 1/3] Nightly cleanup --- .github/workflows/_deb_build.yml | 6 +- .github/workflows/pull_request_closed.yml | 8 +- .github/workflows/release_cleanup.yml | 49 ++++++ scripts/ci-admin | 196 +++++++++++++++------- 4 files changed, 196 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/release_cleanup.yml diff --git a/.github/workflows/_deb_build.yml b/.github/workflows/_deb_build.yml index e771a1c..cc34b21 100644 --- a/.github/workflows/_deb_build.yml +++ b/.github/workflows/_deb_build.yml @@ -82,11 +82,11 @@ jobs: deb_builder_tag=$(echo ${{inputs.base-tag}} | tr : - | tr / -) case "${{ inputs.pull-request }}" in true) - rti_license_file=${{ github.workspace }}/src/uno/rti_license.dat + rti_license_file=src/uno/rti_license.dat base_image=ghcr.io/mentalsmash/uno-ci-base-tester:${deb_builder_tag} ;; false) - rti_license_file=${{ github.workspace }}/src/uno-ci/docker/base-tester/resource/rti/rti_license.dat + rti_license_file=src/uno-ci/docker/base-tester/resource/rti/rti_license.dat base_image=${{ inputs.base-tag }} ;; esac @@ -161,7 +161,7 @@ jobs: env: DEB_TESTER: ${{ needs.config.outputs.TEST_IMAGE }} FIX_DIR: ${{ github.workspace }} - RTI_LICENSE_FILE: ${{ needs.config.outputs.RTI_LICENSE_FILE }} + RTI_LICENSE_FILE: ${{ github.workspace }}/${{ needs.config.outputs.RTI_LICENSE_FILE }} TEST_DATE: ${{ needs.config.outputs.TEST_DATE }} TEST_ID: ${{ needs.config.outputs.TEST_ID }} TEST_IMAGE: ${{ needs.config.outputs.TEST_IMAGE }} diff --git a/.github/workflows/pull_request_closed.yml b/.github/workflows/pull_request_closed.yml index 9dcf50b..46a3d8b 100644 --- a/.github/workflows/pull_request_closed.yml +++ b/.github/workflows/pull_request_closed.yml @@ -29,6 +29,10 @@ permissions: jobs: cleanup_jobs: runs-on: ubuntu-latest + # Run automatically only if the PR is from this repository, + # otherwise it won't be authorized to delete workflow runs. + # (suggested by https://github.com/orgs/community/discussions/25217) + if: github.event.pull_request.head.repo.full_name == github.repository steps: - name: Clone uno uses: actions/checkout@v4 @@ -55,8 +59,6 @@ jobs: -N ${{ github.event_name == 'pull_request' && github.event.pull_request.number || inputs.pr-number }} \ ${{ (github.event_name == 'pull_request' && github.event.pull_request.merged || inputs.pr-merged) && '-m' || '' }} env: - # Changes to actions (e.g. delete workflow runs) require a PAT, and don't work with the GITHUB_TOKEN - # see https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#delete-a-workflow-run - GH_TOKEN: ${{ secrets.UNO_CI_ADMIN_PAT }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ADMIN_IMAGE: ghcr.io/mentalsmash/uno-ci-admin:latest diff --git a/.github/workflows/release_cleanup.yml b/.github/workflows/release_cleanup.yml new file mode 100644 index 0000000..e6dc8bb --- /dev/null +++ b/.github/workflows/release_cleanup.yml @@ -0,0 +1,49 @@ +name: Release (Cleanup) +run-name: | + release cleanup [${{github.ref_type == 'tag' && 'stable' || 'nightly' }}, ${{github.ref_name}}] + +on: + workflow_dispatch: + + workflow_run: + workflows: ["Release"] + types: [completed] + +concurrency: + group: release-cleanup-${{ github.ref }} + cancel-in-progress: false + +permissions: + actions: write + packages: read + +jobs: + cleanup_jobs: + runs-on: ubuntu-latest + steps: + - name: Clone uno + uses: actions/checkout@v4 + with: + path: src/uno + submodules: true + + - name: Log in to GitHub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: "Clean up workflow runs" + run: | + docker run --rm \ + -v $(pwd):/workspace \ + -e GH_TOKEN=${GH_TOKEN} \ + -w /workspace \ + ${ADMIN_IMAGE} \ + src/uno/scripts/ci-admin nightly-cleanup \ + -r ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ADMIN_IMAGE: ghcr.io/mentalsmash/uno-ci-admin:latest + diff --git a/scripts/ci-admin b/scripts/ci-admin index 1f27254..b1b374e 100755 --- a/scripts/ci-admin +++ b/scripts/ci-admin @@ -60,8 +60,6 @@ import re import sys import subprocess import argparse -import tempfile -import contextlib import traceback from functools import partial from pathlib import Path @@ -269,11 +267,14 @@ class DataObject: @classmethod def build(cls, obj_cls: type[NamedTuple], *args) -> NamedTuple: cls.data_object(obj_cls) - args = list(args) - for i in obj_cls.DatetimeFields: - if not isinstance(args[i], datetime): - args[i] = github_date_parse(args[i]) - return obj_cls(*args) + build_args = list(args) + for i, arg in enumerate(args): + if i in obj_cls.DatetimeFields: + if not isinstance(arg, datetime): + build_args[i] = github_date_parse(arg) + elif isinstance(arg, str): + build_args[i] = arg.strip() + return obj_cls(*build_args) @classmethod def parse(cls, obj_cls: type[NamedTuple], package_line: str) -> object | None: @@ -326,13 +327,36 @@ def build(obj_cls: type[NamedTuple], *args) -> object | None: # GitHub Workflow Run data object (parsed from query result) ############################################################################### class WorkflowRun(NamedTuple): - outcome: str - date: datetime id: int - event: str name: str + event: str + status: str + outcome: str + created_at: datetime + updated_at: datetime - DatetimeFields = [1] + DatetimeFields = [5, 6] + SelectQuery = """\ +def symbol: + sub(""; "")? // "NULL" | + sub("skipped"; "SKIP") | + sub("success"; "GOOD") | + sub("startup_failure"; "FAIL") | + sub("cancelled"; "FAIL") | + sub("failure"; "FAIL"); + +[ .workflow_runs[] + | [ + .id, + .name, + .event, + .status, + (.conclusion | symbol), + .created_at, + .updated_at + ] + ] +""" def __str__(self) -> str: return DataObject.str(self) @@ -355,33 +379,6 @@ class WorkflowRun(NamedTuple): prompt: str | None = None, noninteractive: bool = False, ) -> list["WorkflowRun"]: - @contextlib.contextmanager - def _jqscript() -> Generator[Path, None, None]: - script = """\ -def symbol: - sub(""; "")? // "NULL" | - sub("skipped"; "SKIP") | - sub("success"; "GOOD") | - sub("startup_failure"; "FAIL") | - sub("cancelled"; "FAIL") | - sub("failure"; "FAIL"); - -[ .workflow_runs[] - | [ - (.conclusion | symbol), - .created_at, - .id, - .event, - .name - ] - ] -""" - - tmp_h = tempfile.NamedTemporaryFile() - script_file = Path(tmp_h.name) - script_file.write_text(script) - yield script_file - def _read_and_parse_runs(input_stream: TextIO) -> list[WorkflowRun]: return [ run @@ -401,17 +398,20 @@ def symbol: with input_file.open("r") as istream: target_runs = _read_and_parse_runs(istream) else: - with _jqscript() as jqscript: - query_cmd = [f"gh api --paginate /repos/{repo}/actions/runs" " | " f"jq -r -f {jqscript}"] - log.command(query_cmd, shell=True, check=True) - result = subprocess.run(query_cmd, shell=True, check=True, stdout=subprocess.PIPE) - target_runs = [] - if result.stdout: - run_entries = json.loads(result.stdout.decode()) - target_runs.extend(DataObject.build(cls, *entry) for entry in run_entries) + read_cmd = ["gh", "api", "--paginate", f"/repos/{repo}/actions/runs"] + log.command(read_cmd) + read_process = subprocess.Popen(read_cmd, stdout=subprocess.PIPE) + jq_cmd = ["jq", WorkflowRun.SelectQuery] + log.command(jq_cmd) + result = subprocess.run(jq_cmd, stdin=read_process.stdout, stdout=subprocess.PIPE, check=True) + target_runs = [] + if result.stdout: + result = result.stdout.decode() + run_entries = json.loads(result) + target_runs.extend(DataObject.build(cls, *entry) for entry in run_entries) if prompt is None: prompt = "available runs" - sorted_runs = partial(sorted, key=lambda r: r.date) + sorted_runs = partial(sorted, key=lambda r: r.created_at) fzf = fzf_filter( filter=filter, inputs=sorted_runs(target_runs), prompt=prompt, noninteractive=noninteractive ) @@ -454,9 +454,9 @@ def symbol: ############################################################################### class Package(NamedTuple): id: str + repository: str name: str visibility: str - repository: str created_at: datetime updated_at: datetime @@ -479,7 +479,7 @@ class Package(NamedTuple): ) -> list["Package"]: def _ls_packages() -> Generator[Package, None, None]: jq_filter = ( - "[ (.[] | [.id, .name, .visibility, .repository.full_name , .created_at, .updated_at]) ]" + "[ (.[] | [.id, .repository.full_name, .name, .visibility, .created_at, .updated_at]) ]" ) url = ( f"/orgs/{org}/packages?package_type={package_type}" @@ -610,18 +610,22 @@ class PackageVersion(NamedTuple): return deleted +############################################################################### +# Combine the result arrays of an action which deletes/keeps workflow runs +############################################################################### +def workflow_run_action_result( + removed: list[WorkflowRun], preserved: list[WorkflowRun] +) -> list[tuple[bool, WorkflowRun]]: + result = [*((True, run) for run in removed), *((False, run) for run in preserved)] + return sorted(result, key=lambda v: v[1].created_at) + + ############################################################################### # Perform cleanup procedures after on a closed Pull Request ############################################################################### def pr_closed( repo: str, pr_no: int, merged: bool, noop: bool = False ) -> list[tuple[bool, WorkflowRun]]: - def _result( - removed: list[WorkflowRun], preserved: list[WorkflowRun] - ) -> list[tuple[bool, WorkflowRun]]: - result = [*((True, run) for run in removed), *((False, run) for run in preserved)] - return sorted(result, key=lambda v: v[1].date) - all_runs = pr_runs(repo, pr_no, noninteractive=True) if not all_runs: @@ -636,7 +640,7 @@ def pr_closed( log.warning("[{}][PR #{}] deleting all {} runs for unmerged PR", pr_no, repo, len(all_runs)) removed = WorkflowRun.delete(repo, noop=noop, runs=all_runs) preserved = [run for run in all_runs if run not in removed] - return _result(removed, preserved) + return workflow_run_action_result(removed, preserved) log.activity("[{}][PR #{}] listing failed and skipped runs", repo, pr_no) removed = list(pr_runs(repo, pr_no, "FAIL | ^SKIP | ^NULL", runs=all_runs, noninteractive=True)) @@ -716,7 +720,7 @@ def pr_closed( else: log.warning("[{}][PR #{}] {} runs ARCHIVED", repo, pr_no, len(preserved)) - return _result(actually_removed, preserved) + return workflow_run_action_result(actually_removed, preserved) ############################################################################### @@ -732,6 +736,73 @@ def pr_runs( return WorkflowRun.select(repo, filter, **select_args) +############################################################################### +# Nightly Release - periodic cleanup +############################################################################### +def nightly_cleanup(repo: str, noop: bool = False) -> None: + preserved = [] + removed = [] + + def _pick_preserved(runs: list[WorkflowRun]) -> list[WorkflowRun]: + latest = runs[-1] + result = [latest] + if latest.outcome != "GOOD": + latest_ok = next((run for run in reversed(runs) if run.outcome == "GOOD"), None) + if latest_ok: + result.append(latest_ok) + return result + + def _scan_runs(run_type: str, filter: str, remove_all: bool = False) -> list[WorkflowRun]: + runs = WorkflowRun.select(repo, filter, noninteractive=True) + if not runs: + log.warning("[{}] no {} detected", repo, run_type) + else: + log.info("[{}] {} {} runs detected", repo, run_type, len(runs)) + for i, run in enumerate(runs): + log.info("[{}] {}. {}", repo, i, run) + if not remove_all: + preserved.extend(_pick_preserved(runs)) + for run in runs: + if run in preserved: + continue + removed.append(run) + return runs + + _ = _scan_runs("nighlty releases", "!'deb 'release !'cleanup '[nightly") + _ = _scan_runs("nighlty DEB releases", "'deb 'release '[nightly") + _ = _scan_runs( + "nighlty release cleanup jobs", "!'deb 'release 'cleanup '[nightly", remove_all=True + ) + + if preserved: + log.info("[{}] {} candidates for ARCHIVAL", repo, len(preserved)) + if not ScriptNoninteractive: + removed.extend(WorkflowRun.select(repo, runs=preserved, prompt="don't archive")) + else: + log.warning("[{}] no candidates for ARCHIVAL", repo) + + if removed: + log.warning("[{}] {} candidates for DELETION", repo, len(removed)) + actually_removed = WorkflowRun.delete(repo, noop=noop, runs=removed) + else: + log.info("[{}] no candidates for DELETION", repo) + actually_removed = [] + + preserved.extend(run for run in removed if run not in actually_removed) + + if not actually_removed: + log.info("[{}] no runs deleted", repo) + else: + log.warning("[{}] {} runs DELETED", repo, len(actually_removed)) + + if not preserved: + log.warning("[{}] no runs archived", repo) + else: + log.warning("[{}] {} runs ARCHIVED", repo, len(preserved)) + + return workflow_run_action_result(removed, preserved) + + ############################################################################### # Command-line arguments parser ############################################################################### @@ -833,6 +904,13 @@ def define_parser() -> argparse.ArgumentParser: "-f", "--filter", help="Custom zfz filter to run in unattended mode.", default=None ) + parser_nightly_cleanup = subparsers.add_parser( + "nightly-cleanup", help="Clean up workflow runs for nightly releases" + ) + parser_nightly_cleanup.add_argument( + "-r", "--repository", help="Target GitHub repository (owner/repo).", required=True + ) + return parser @@ -879,6 +957,10 @@ def dispatch_action(args: argparse.Namespace) -> None: noop=args.noop, ): output(str(run)) + elif args.action == "nightly-cleanup": + tabulate_columns("action", *WorkflowRun._fields) + for removed, run in nightly_cleanup(repo=args.repository, noop=args.noop): + output("DEL" if removed else "KEEP", str(run)) else: raise RuntimeError("action not implemented", args.action) finally: From 5b1b3a691db25a370397d8c46bb48f916e06b8d4 Mon Sep 17 00:00:00 2001 From: Andrea Sorbini Date: Wed, 17 Apr 2024 23:50:38 -0700 Subject: [PATCH 2/3] More resiliant PR handling --- scripts/ci-admin | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/ci-admin b/scripts/ci-admin index b1b374e..570e4d0 100755 --- a/scripts/ci-admin +++ b/scripts/ci-admin @@ -328,14 +328,14 @@ def build(obj_cls: type[NamedTuple], *args) -> object | None: ############################################################################### class WorkflowRun(NamedTuple): id: int - name: str + created_at: datetime + updated_at: datetime event: str status: str outcome: str - created_at: datetime - updated_at: datetime + name: str - DatetimeFields = [5, 6] + DatetimeFields = [1, 2] SelectQuery = """\ def symbol: sub(""; "")? // "NULL" | @@ -348,12 +348,12 @@ def symbol: [ .workflow_runs[] | [ .id, - .name, + .created_at, + .updated_at, .event, .status, (.conclusion | symbol), - .created_at, - .updated_at + .name ] ] """ @@ -643,7 +643,7 @@ def pr_closed( return workflow_run_action_result(removed, preserved) log.activity("[{}][PR #{}] listing failed and skipped runs", repo, pr_no) - removed = list(pr_runs(repo, pr_no, "FAIL | ^SKIP | ^NULL", runs=all_runs, noninteractive=True)) + removed = list(pr_runs(repo, pr_no, "'FAIL | 'SKIP | 'NULL", runs=all_runs, noninteractive=True)) if not removed: log.info("[{}][PR #{}] no failed nor skipped runs", repo, pr_no) else: @@ -653,7 +653,7 @@ def pr_closed( log.activity("[{}][PR #{}] listing good 'basic validation' runs", repo, pr_no) basic_validation_runs = list( - pr_runs(repo, pr_no, "GOOD", "updated", runs=all_runs, noninteractive=True) + pr_runs(repo, pr_no, "'GOOD", "updated", runs=all_runs, noninteractive=True) ) if not basic_validation_runs: log.warning("[{}][PR #{}] no good 'basic validation' run", repo, pr_no) @@ -674,7 +674,7 @@ def pr_closed( log.activity("[{}][PR #{}] listing good 'full validation' runs", repo, pr_no) full_validation_runs = list( - pr_runs(repo, pr_no, "GOOD", "reviewed, approved", runs=all_runs, noninteractive=True) + pr_runs(repo, pr_no, "'GOOD", "reviewed, 'approved", runs=all_runs, noninteractive=True) ) if not full_validation_runs: log.error("[{}][PR #{}] no good 'full validation' run!", repo, pr_no) @@ -730,7 +730,7 @@ def pr_runs( repo: str, pr_no: int, result: str | None = None, category: str | None = None, **select_args ) -> Generator[WorkflowRun, None, None]: filter = ( - f"{'^'+result+' ' if result else ''}'PR '#{pr_no} '[{category if category is not None else ''}" + f"{result+' ' if result else ''}'PR '#{pr_no} '[{category if category is not None else ''}" ) select_args.setdefault("prompt", f"runs for PR #{pr_no}") return WorkflowRun.select(repo, filter, **select_args) From 01ae9795d7e8f19721b76075ee25b76ba3c76fe7 Mon Sep 17 00:00:00 2001 From: Andrea Sorbini Date: Wed, 17 Apr 2024 23:52:21 -0700 Subject: [PATCH 3/3] Only run release cleanup on successful runs --- .github/workflows/release_cleanup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_cleanup.yml b/.github/workflows/release_cleanup.yml index e6dc8bb..da20c69 100644 --- a/.github/workflows/release_cleanup.yml +++ b/.github/workflows/release_cleanup.yml @@ -19,6 +19,7 @@ permissions: jobs: cleanup_jobs: + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest steps: - name: Clone uno