From 6b7959e18acf4ad1e9b0ad6a2f4dc0fbb9e90173 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Wed, 22 Nov 2023 10:23:46 +0100 Subject: [PATCH 01/20] Self hosted runners: make VM image configurable through environment variable The image SKU was hidden a bit in the azure-arm-template.json. Let's make it more visible by turning it into a parameter that the CI pipeline can provide. Signed-off-by: Dennis Ameling --- .github/workflows/create-azure-self-hosted-runners.yml | 2 ++ azure-self-hosted-runners/azure-arm-template.json | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-azure-self-hosted-runners.yml b/.github/workflows/create-azure-self-hosted-runners.yml index b9316d2a..dd6d9e19 100644 --- a/.github/workflows/create-azure-self-hosted-runners.yml +++ b/.github/workflows/create-azure-self-hosted-runners.yml @@ -39,6 +39,7 @@ env: # For more information, see https://learn.microsoft.com/en-us/azure/virtual-machines/dplsv5-dpldsv5-series (which # unfortunately does not have more information about price by region) AZURE_VM_REGION: westus2 + AZURE_VM_IMAGE: win11-22h2-ent # The following secrets are required for this workflow to run: # AZURE_CREDENTIALS - Credentials for the Azure CLI. It's recommended to set up a resource @@ -130,6 +131,7 @@ jobs: githubActionsRunnerRegistrationUrl="$ACTIONS_RUNNER_REGISTRATION_URL" githubActionsRunnerToken="$ACTIONS_RUNNER_TOKEN" postDeploymentPsScriptUrl="$POST_DEPLOYMENT_SCRIPT_URL" + virtualMachineImage="$AZURE_VM_IMAGE" virtualMachineName="${{ steps.generate-vm-name.outputs.vm_name }}" virtualMachineSize="$AZURE_VM_TYPE" publicIpAddressName1="${{ steps.generate-vm-name.outputs.vm_name }}-ip" diff --git a/azure-self-hosted-runners/azure-arm-template.json b/azure-self-hosted-runners/azure-arm-template.json index 42a81bad..1bf6b140 100644 --- a/azure-self-hosted-runners/azure-arm-template.json +++ b/azure-self-hosted-runners/azure-arm-template.json @@ -81,6 +81,9 @@ "osDiskDeleteOption": { "type": "string" }, + "virtualMachineImage": { + "type": "string" + }, "virtualMachineSize": { "type": "string" }, @@ -209,7 +212,7 @@ "imageReference": { "publisher": "microsoftwindowsdesktop", "offer": "windows11preview-arm64", - "sku": "win11-22h2-ent", + "sku": "[parameters('virtualMachineImage')]", "version": "latest" } }, From 59003a98f3fd0ac7c2c6ef95d35843609610f3fe Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Wed, 22 Nov 2023 10:28:32 +0100 Subject: [PATCH 02/20] Self hosted runners: update image from 22h2 to 23h2 Version 23h2 was released on October 31st, 2023. Let's update to it, so that the runners benefit from the latest improvements to Windows 11 ARM64. Signed-off-by: Dennis Ameling --- .github/workflows/create-azure-self-hosted-runners.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-azure-self-hosted-runners.yml b/.github/workflows/create-azure-self-hosted-runners.yml index dd6d9e19..e60ddf4b 100644 --- a/.github/workflows/create-azure-self-hosted-runners.yml +++ b/.github/workflows/create-azure-self-hosted-runners.yml @@ -39,7 +39,7 @@ env: # For more information, see https://learn.microsoft.com/en-us/azure/virtual-machines/dplsv5-dpldsv5-series (which # unfortunately does not have more information about price by region) AZURE_VM_REGION: westus2 - AZURE_VM_IMAGE: win11-22h2-ent + AZURE_VM_IMAGE: win11-23h2-ent # The following secrets are required for this workflow to run: # AZURE_CREDENTIALS - Credentials for the Azure CLI. It's recommended to set up a resource From f428d14f1fbeb550d1f56c4413fab7b9cb205ee6 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Wed, 22 Nov 2023 11:26:06 +0100 Subject: [PATCH 03/20] Self hosted runners: add cleanup script Sometimes, VMs don't get deleted properly. Let's set up a script to clean up VMs that were created more than 3 hours ago. This script runs every 6 hours (cron) and we can also invoke it manually if needed. Signed-off-by: Dennis Ameling --- .../workflows/cleanup-self-hosted-runners.yml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/cleanup-self-hosted-runners.yml diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml new file mode 100644 index 00000000..d365764f --- /dev/null +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -0,0 +1,56 @@ +name: Cleanup Azure self hosted runners +run-name: Cleanup Azure self hosted runners + +on: + schedule: + # Run every 6 hours + - cron: "0 */6 * * *" + workflow_dispatch: + +# The following secrets are required for this workflow to run: +# AZURE_CREDENTIALS - Credentials for the Azure CLI. It's recommended to set up a resource +# group specifically for self-hosted Actions Runners. +# az ad sp create-for-rbac --name "{YOUR_DESCRIPTIVE_NAME_HERE}" --role contributor \ +# --scopes /subscriptions/{SUBSCRIPTION_ID_HERE}/resourceGroups/{RESOURCE_GROUP_HERE} \ +# --sdk-auth +# AZURE_RESOURCE_GROUP - Resource group to create the runner(s) in +jobs: + delete-runner: + runs-on: ubuntu-latest + steps: + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Discover VMs to delete + uses: azure/CLI@v1 + with: + azcliversion: 2.54.0 + inlineScript: | + active_vms=$(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }} | jq -c '.[] | {name,timeCreated}') + current_time=$(date +%s) + three_hours_ago=$(($current_time - 3 * 3600)) + + if [ -z "$active_vms" ]; then + echo "No active VMs found, nothing to do." + exit 0 + else + echo "Found these active VMs:" + echo $active_vms + fi + + for active_vm in ${active_vms[@]}; do + vm_name=$(echo $active_vm | jq '.name') + vm_creation_iso_string=$(echo $active_vm | jq -r '.timeCreated') + vm_creation_time=$(date -d $vm_creation_iso_string +%s) + + if [ "$three_hours_ago" -ge "$vm_creation_time" ]; then + echo "The VM ${vm_name} was created more than 3 hours ago and wasn't deleted. Let's do that now." + az vm delete -n "$vm_name" -g ${{ secrets.AZURE_RESOURCE_GROUP }} --yes + az network nsg delete -n "$vm_name"-nsg -g ${{ secrets.AZURE_RESOURCE_GROUP }} + az network vnet delete -n "$vm_name"-vnet -g ${{ secrets.AZURE_RESOURCE_GROUP }} + az network public-ip delete -n "$vm_name"-ip -g ${{ secrets.AZURE_RESOURCE_GROUP }} + else + echo "The VM ${vm_name} was created less then 3 hours ago and shouldn't be deleted yet. Skipping." + fi + done From 7bd6ad388b8b2ce348983088adc5950349142e5f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Sep 2024 13:00:57 +0200 Subject: [PATCH 04/20] cleanup-self-hosted-runners: use a newer GitHub CLI version Signed-off-by: Johannes Schindelin --- .github/workflows/cleanup-self-hosted-runners.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index d365764f..e65bec10 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -25,7 +25,7 @@ jobs: - name: Discover VMs to delete uses: azure/CLI@v1 with: - azcliversion: 2.54.0 + azcliversion: 2.64.0 inlineScript: | active_vms=$(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }} | jq -c '.[] | {name,timeCreated}') current_time=$(date +%s) From 91c4017e644f6cd98ba005e8e506fdf11e8432c7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Sep 2024 11:01:18 +0200 Subject: [PATCH 05/20] cleanup-self-hosted-runner: order the conditional blocks from small to large In general, if..else..fi constructs are more readable when the first block is shorter than the second, i.e. to handle the "easy" things first; According to Cognitive Load Theory, this uses less mental resources when wrapping our head around the code. In this instance, there is an even better reason: The next commit will add _another_ condition, and having it in this order will avoid adding another nesting level: if too young skip else if busy skip else force delete VM fi Signed-off-by: Johannes Schindelin --- .github/workflows/cleanup-self-hosted-runners.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index e65bec10..b099ec1e 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -44,13 +44,13 @@ jobs: vm_creation_iso_string=$(echo $active_vm | jq -r '.timeCreated') vm_creation_time=$(date -d $vm_creation_iso_string +%s) - if [ "$three_hours_ago" -ge "$vm_creation_time" ]; then + if [ "$three_hours_ago" -lt "$vm_creation_time" ]; then + echo "The VM ${vm_name} was created less then 3 hours ago and shouldn't be deleted yet. Skipping." + else echo "The VM ${vm_name} was created more than 3 hours ago and wasn't deleted. Let's do that now." az vm delete -n "$vm_name" -g ${{ secrets.AZURE_RESOURCE_GROUP }} --yes az network nsg delete -n "$vm_name"-nsg -g ${{ secrets.AZURE_RESOURCE_GROUP }} az network vnet delete -n "$vm_name"-vnet -g ${{ secrets.AZURE_RESOURCE_GROUP }} az network public-ip delete -n "$vm_name"-ip -g ${{ secrets.AZURE_RESOURCE_GROUP }} - else - echo "The VM ${vm_name} was created less then 3 hours ago and shouldn't be deleted yet. Skipping." fi done From cb7c524828d03e303a4b636cf26a9808e253b249 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Sep 2024 12:17:57 +0200 Subject: [PATCH 06/20] Add a shell script to authenticate with `gh` as an App In Git for Windows' automation, we do a lot with Javascript, but sometimes it is also convenient to use the GitHub CLI (`gh`) in a shell script. Sadly, the scope of the `GITHUB_TOKEN` provided in GitHub workflows is often too limited, and we'd like to authenticate as a GitHub App instead. Even more sadly, authenticating with `gh` this way is quite complicated and fraught with problems because the token _needs_ to be masked in the GitHub workflow logs. Rejoice! This patch brings a shell script that hides all that nasty complexity. All it needs are the environment variables: - GH_APP_ID - GH_APP_PRIVATE_KEY - GITHUB_REPOSITORY and of course `gh` on the `PATH`. That's it! Signed-off-by: Johannes Schindelin --- gh-cli-auth-as-app.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 gh-cli-auth-as-app.sh diff --git a/gh-cli-auth-as-app.sh b/gh-cli-auth-as-app.sh new file mode 100755 index 00000000..5641f982 --- /dev/null +++ b/gh-cli-auth-as-app.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +node -e '(async () => { + const [owner, repo] = process.env.GITHUB_REPOSITORY.split("/") + const getAppInstallationId = require("./get-app-installation-id") + const installationId = await getAppInstallationId( + console, + process.env.GH_APP_ID, + process.env.GH_APP_PRIVATE_KEY, + owner, + repo + ) + const getInstallationAccessToken = require("./get-installation-access-token") + const token = await getInstallationAccessToken( + console, + process.env.GH_APP_ID, + process.env.GH_APP_PRIVATE_KEY, + installationId + ) + process.stderr.write(`::add-mask::${token.token}\n`) + process.stdout.write(token.token) +})().catch(e => { + process.stderr.write(JSON.stringify(e, null, 2)) + process.exit(1) +})' | gh auth login --with-token From 692b6f41d5c9a952addbf359bf4ad2b2730931b8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Sep 2024 11:12:47 +0200 Subject: [PATCH 07/20] cleanup-self-hosted-runner: avoid deleting VMs that are busy So far, we avoid deleting VMs when they are too young. But we can do better than that: since the VMs are intended to host an ephemeral runner, and that runner is registered with the current repository, we can have a look whether GitHub says that the runner is busy. If it is, well, let's leave it a-running! Since this query requires a larger scope than the standard GitHub workflows' `GITHUB_TOKEN` provides, we need to authenticate as the GitForWindowsHelper GitHub App to do that. This is too complicated, unfortunately, in a GitHub workflow (at least without the risk of using non-official GitHub Actions), therefore we use the newly-added shell script to authenticate the GitHub CLI as a GitHub App (which of course now requires us to check out the repository). Signed-off-by: Johannes Schindelin --- .github/workflows/cleanup-self-hosted-runners.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index b099ec1e..14a54ebc 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -18,12 +18,16 @@ jobs: delete-runner: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 - name: Azure Login uses: azure/login@v1 with: creds: ${{ secrets.AZURE_CREDENTIALS }} - name: Discover VMs to delete uses: azure/CLI@v1 + env: + GH_APP_ID: ${{ secrets.GH_APP_ID }} + GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} with: azcliversion: 2.64.0 inlineScript: | @@ -46,6 +50,13 @@ jobs: if [ "$three_hours_ago" -lt "$vm_creation_time" ]; then echo "The VM ${vm_name} was created less then 3 hours ago and shouldn't be deleted yet. Skipping." + elif test true = "$(if test ! -f .cli-authenticated; then + ./gh-cli-auth-as-app.sh && + >.cli-authenticated # only authenticate once + fi && + gh api repos/$GITHUB_REPOSITORY/actions/runners \ + --jq '.runners[] | select(.name == "'$vm_name'") | .busy)"; then + echo "The VM ${vm_name} is still busy." else echo "The VM ${vm_name} was created more than 3 hours ago and wasn't deleted. Let's do that now." az vm delete -n "$vm_name" -g ${{ secrets.AZURE_RESOURCE_GROUP }} --yes From 55fbe54dedefc426fb11e1f4b80892f932bac449 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Sep 2024 11:16:37 +0200 Subject: [PATCH 08/20] cleanup-self-hosted-runner: clean up idle runners earlier The original idea was to wait at least 3 hours before deleting a VM, to avoid interfering with long-running workflows. Now that we have a way to query whether the runner _is_ busy, we do not need to wait that long. Basically, we only need to ensure that the runner had a chance to register itself and to then pick up the job. Typically, this takes around 6-7 minutes, so waiting for an hour is probably even a tad conservative. Signed-off-by: Johannes Schindelin --- .github/workflows/cleanup-self-hosted-runners.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 14a54ebc..f347fb73 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -33,7 +33,7 @@ jobs: inlineScript: | active_vms=$(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }} | jq -c '.[] | {name,timeCreated}') current_time=$(date +%s) - three_hours_ago=$(($current_time - 3 * 3600)) + one_hour_ago=$(($current_time - 3600)) if [ -z "$active_vms" ]; then echo "No active VMs found, nothing to do." @@ -48,8 +48,8 @@ jobs: vm_creation_iso_string=$(echo $active_vm | jq -r '.timeCreated') vm_creation_time=$(date -d $vm_creation_iso_string +%s) - if [ "$three_hours_ago" -lt "$vm_creation_time" ]; then - echo "The VM ${vm_name} was created less then 3 hours ago and shouldn't be deleted yet. Skipping." + if [ "$one_hour_ago" -lt "$vm_creation_time" ]; then + echo "The VM ${vm_name} was created less then 1 hour ago and shouldn't be deleted yet. Skipping." elif test true = "$(if test ! -f .cli-authenticated; then ./gh-cli-auth-as-app.sh && >.cli-authenticated # only authenticate once From ded928ed0fbd97278d80b16f9049e87faa5e14c4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Sep 2024 11:21:07 +0200 Subject: [PATCH 09/20] cleanup-self-hosted-runner: fix some white-space issue Signed-off-by: Johannes Schindelin --- .github/workflows/cleanup-self-hosted-runners.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index f347fb73..7319c545 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -47,7 +47,7 @@ jobs: vm_name=$(echo $active_vm | jq '.name') vm_creation_iso_string=$(echo $active_vm | jq -r '.timeCreated') vm_creation_time=$(date -d $vm_creation_iso_string +%s) - + if [ "$one_hour_ago" -lt "$vm_creation_time" ]; then echo "The VM ${vm_name} was created less then 1 hour ago and shouldn't be deleted yet. Skipping." elif test true = "$(if test ! -f .cli-authenticated; then From 8acd97b982a85bb9076bef6b01811a18093ccd25 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Sep 2024 13:05:37 +0200 Subject: [PATCH 10/20] cleanup-self-hosted-runner: use workflow commands for better mark-up When logging output that we're deleting a VM, it should be marked a bit more prominently. Let's mark it as a warning: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-warning-message This marks it in red in the output. Conversely, we want to mention when we skipped the deletion, and to make those messages stick out a bit (but less than warnings), let's use: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-notice-message Signed-off-by: Johannes Schindelin --- .github/workflows/cleanup-self-hosted-runners.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 7319c545..7c38c046 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -49,16 +49,16 @@ jobs: vm_creation_time=$(date -d $vm_creation_iso_string +%s) if [ "$one_hour_ago" -lt "$vm_creation_time" ]; then - echo "The VM ${vm_name} was created less then 1 hour ago and shouldn't be deleted yet. Skipping." + echo "::notice::The VM ${vm_name} was created less then 1 hour ago and shouldn't be deleted yet. Skipping." elif test true = "$(if test ! -f .cli-authenticated; then ./gh-cli-auth-as-app.sh && >.cli-authenticated # only authenticate once fi && gh api repos/$GITHUB_REPOSITORY/actions/runners \ --jq '.runners[] | select(.name == "'$vm_name'") | .busy)"; then - echo "The VM ${vm_name} is still busy." + echo "::notice::The VM ${vm_name} is still busy." else - echo "The VM ${vm_name} was created more than 3 hours ago and wasn't deleted. Let's do that now." + echo "::warning::The VM ${vm_name} was created more than 3 hours ago and wasn't deleted. Let's do that now." az vm delete -n "$vm_name" -g ${{ secrets.AZURE_RESOURCE_GROUP }} --yes az network nsg delete -n "$vm_name"-nsg -g ${{ secrets.AZURE_RESOURCE_GROUP }} az network vnet delete -n "$vm_name"-vnet -g ${{ secrets.AZURE_RESOURCE_GROUP }} From 2c67ce62194d9f368d6cac0bac0ac2cfb66cda7e Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 13:33:04 +0200 Subject: [PATCH 11/20] TO-DROP: run on push --- .github/workflows/cleanup-self-hosted-runners.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 7c38c046..62ab2b61 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -6,6 +6,7 @@ on: # Run every 6 hours - cron: "0 */6 * * *" workflow_dispatch: + push: # The following secrets are required for this workflow to run: # AZURE_CREDENTIALS - Credentials for the Azure CLI. It's recommended to set up a resource From 89a5352c142a033d05b4f62216b8c4e3b00392cb Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 13:33:57 +0200 Subject: [PATCH 12/20] TO-DROP: run on pull_request instead of push --- .github/workflows/cleanup-self-hosted-runners.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 62ab2b61..51f83d00 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -6,7 +6,7 @@ on: # Run every 6 hours - cron: "0 */6 * * *" workflow_dispatch: - push: + pull_request: # The following secrets are required for this workflow to run: # AZURE_CREDENTIALS - Credentials for the Azure CLI. It's recommended to set up a resource From 332a05d7fab81efe288595557042eff280bb094f Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 13:39:29 +0200 Subject: [PATCH 13/20] Azure Actions: upgrade to v2 and pin Azure CLI to 2.63.0 --- .github/workflows/cleanup-self-hosted-runners.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 51f83d00..577500dd 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -21,16 +21,17 @@ jobs: steps: - uses: actions/checkout@v4 - name: Azure Login - uses: azure/login@v1 + uses: azure/login@v2 with: creds: ${{ secrets.AZURE_CREDENTIALS }} - name: Discover VMs to delete - uses: azure/CLI@v1 + uses: azure/CLI@v2 env: GH_APP_ID: ${{ secrets.GH_APP_ID }} GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} with: - azcliversion: 2.64.0 + # Stick to 2.63.0 until jq is added to 2.64.0+ https://github.com/Azure/azure-cli/issues/29830 + azcliversion: 2.63.0 inlineScript: | active_vms=$(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }} | jq -c '.[] | {name,timeCreated}') current_time=$(date +%s) From 1fcd7a8a8b06c286f2f21d33b8d7e8b95095cf49 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 14:06:18 +0200 Subject: [PATCH 14/20] set -x for debugging --- .github/workflows/cleanup-self-hosted-runners.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 577500dd..6c4175e9 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -33,6 +33,7 @@ jobs: # Stick to 2.63.0 until jq is added to 2.64.0+ https://github.com/Azure/azure-cli/issues/29830 azcliversion: 2.63.0 inlineScript: | + set -x active_vms=$(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }} | jq -c '.[] | {name,timeCreated}') current_time=$(date +%s) one_hour_ago=$(($current_time - 3600)) From 577630b88937dc6aa8665d3b2dfcec589ead991a Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 15:55:52 +0200 Subject: [PATCH 15/20] Fix missing quote in jq command --- .github/workflows/cleanup-self-hosted-runners.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 6c4175e9..26b4a9a6 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -58,7 +58,7 @@ jobs: >.cli-authenticated # only authenticate once fi && gh api repos/$GITHUB_REPOSITORY/actions/runners \ - --jq '.runners[] | select(.name == "'$vm_name'") | .busy)"; then + --jq '.runners[] | select(.name == "'$vm_name'") | .busy')"; then echo "::notice::The VM ${vm_name} is still busy." else echo "::warning::The VM ${vm_name} was created more than 3 hours ago and wasn't deleted. Let's do that now." From b90b2273dae091a4dfbfface1dc786ebeb431177 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 16:03:24 +0200 Subject: [PATCH 16/20] Trim ISO date string --- .github/workflows/cleanup-self-hosted-runners.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 26b4a9a6..d9928d8b 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -49,7 +49,9 @@ jobs: for active_vm in ${active_vms[@]}; do vm_name=$(echo $active_vm | jq '.name') vm_creation_iso_string=$(echo $active_vm | jq -r '.timeCreated') - vm_creation_time=$(date -d $vm_creation_iso_string +%s) + # Bash's date -d command doesn't support the ISO 8601 format with a colon in the timezone offset, so we need to remove it + vm_creation_trimmed=$(echo "$vm_creation_iso_string" | sed 's/\+\([0-9]*\):\([0-9]*\)$//') + vm_creation_time=$(date -d $vm_creation_trimmed +%s) if [ "$one_hour_ago" -lt "$vm_creation_time" ]; then echo "::notice::The VM ${vm_name} was created less then 1 hour ago and shouldn't be deleted yet. Skipping." From cc19e05c79ccb882f2411de623c8e0ab642ff727 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 16:05:38 +0200 Subject: [PATCH 17/20] Another attempt to fix things --- .github/workflows/cleanup-self-hosted-runners.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index d9928d8b..af32ca0e 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -50,8 +50,8 @@ jobs: vm_name=$(echo $active_vm | jq '.name') vm_creation_iso_string=$(echo $active_vm | jq -r '.timeCreated') # Bash's date -d command doesn't support the ISO 8601 format with a colon in the timezone offset, so we need to remove it - vm_creation_trimmed=$(echo "$vm_creation_iso_string" | sed 's/\+\([0-9]*\):\([0-9]*\)$//') - vm_creation_time=$(date -d $vm_creation_trimmed +%s) + vm_creation_trimmed=$(echo "$vm_creation_iso_string" | sed 's/+[^+]*$//') + vm_creation_time=$(TZ=UTC date -d $vm_creation_trimmed +%s) if [ "$one_hour_ago" -lt "$vm_creation_time" ]; then echo "::notice::The VM ${vm_name} was created less then 1 hour ago and shouldn't be deleted yet. Skipping." From 070a1610fb0612a91ca0b9fb30c40e5918d03d4f Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 16:08:36 +0200 Subject: [PATCH 18/20] Another attempt to fix --- .github/workflows/cleanup-self-hosted-runners.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index af32ca0e..6a0c285e 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -49,8 +49,8 @@ jobs: for active_vm in ${active_vms[@]}; do vm_name=$(echo $active_vm | jq '.name') vm_creation_iso_string=$(echo $active_vm | jq -r '.timeCreated') - # Bash's date -d command doesn't support the ISO 8601 format with a colon in the timezone offset, so we need to remove it - vm_creation_trimmed=$(echo "$vm_creation_iso_string" | sed 's/+[^+]*$//') + # Bash's date -d command doesn't support the full ISO 8601 format with milliseconds and timezone, so let's strip those + vm_creation_trimmed=$(echo "$vm_creation_iso_string" | sed 's/\.[0-9]*//;s/+[^+]*$//') vm_creation_time=$(TZ=UTC date -d $vm_creation_trimmed +%s) if [ "$one_hour_ago" -lt "$vm_creation_time" ]; then From 300182d2e4cdf1ae74771a4893b95c1d84e28f59 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 16:17:58 +0200 Subject: [PATCH 19/20] Another attempt to fix --- .github/workflows/cleanup-self-hosted-runners.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 6a0c285e..126634f4 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -48,10 +48,9 @@ jobs: for active_vm in ${active_vms[@]}; do vm_name=$(echo $active_vm | jq '.name') - vm_creation_iso_string=$(echo $active_vm | jq -r '.timeCreated') - # Bash's date -d command doesn't support the full ISO 8601 format with milliseconds and timezone, so let's strip those - vm_creation_trimmed=$(echo "$vm_creation_iso_string" | sed 's/\.[0-9]*//;s/+[^+]*$//') - vm_creation_time=$(TZ=UTC date -d $vm_creation_trimmed +%s) + # Use jq to extract and format the date-time string + vm_creation_time_string=$(echo $active_vm | jq -r '.timeCreated | sub("\\.[0-9]+[+-][0-9]+:[0-9]+$"; "") | sub("T"; " ")') + vm_creation_time=$(TZ=UTC date -d $vm_creation_time_string +%s) if [ "$one_hour_ago" -lt "$vm_creation_time" ]; then echo "::notice::The VM ${vm_name} was created less then 1 hour ago and shouldn't be deleted yet. Skipping." From a99784ae81565fc097b8bd7facce6488e2e74854 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 15 Sep 2024 16:19:31 +0200 Subject: [PATCH 20/20] Add quotes --- .github/workflows/cleanup-self-hosted-runners.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 126634f4..a6ce8a50 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -50,7 +50,7 @@ jobs: vm_name=$(echo $active_vm | jq '.name') # Use jq to extract and format the date-time string vm_creation_time_string=$(echo $active_vm | jq -r '.timeCreated | sub("\\.[0-9]+[+-][0-9]+:[0-9]+$"; "") | sub("T"; " ")') - vm_creation_time=$(TZ=UTC date -d $vm_creation_time_string +%s) + vm_creation_time=$(TZ=UTC date -d "$vm_creation_time_string" +%s) if [ "$one_hour_ago" -lt "$vm_creation_time" ]; then echo "::notice::The VM ${vm_name} was created less then 1 hour ago and shouldn't be deleted yet. Skipping."