Skip to content

Commit a09d928

Browse files
norio-nomuranorio-nomura
committed
setup_cache_for_template: Improve caching mechanism for images and nerdctl archives
- Change cache key to `url-sha256:$sha256` for caching images without a digest - Include image basename in the cache key - Use `limactl validate --fill` to retrieve nerdctl archive info and set cache if needed - `test.yml`: Change cache configuration to run after `make install` Signed-off-by: Norio Nomura <[email protected]>
1 parent 402ba1c commit a09d928

File tree

2 files changed

+110
-34
lines changed

2 files changed

+110
-34
lines changed

‎.github/actions/setup_cache_for_template/action.yml

+93-18
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ inputs:
77
template:
88
description: template yaml file
99
required: true
10+
detect-containerd:
11+
description: detect containerd usage from template by using limactl validate
12+
required: false
13+
default: 'true'
1014
runs:
1115
using: "composite"
1216
steps:
@@ -27,10 +31,15 @@ runs:
2731
set -eux
2832
arch="${{ inputs.arch }}"
2933
template="${{ inputs.template }}"
34+
detect_containerd="${{ inputs.detect-containerd }}"
35+
if [[ $detect_containerd == "true" ]] && ! command -v limactl &>/dev/null; then
36+
echo "containerd detection is disabled because limactl is not found" >&2
37+
detect_containerd="false"
38+
fi
3039
case "$template" in
3140
https://*)
3241
tmp_yaml=$(mktemp -d)/template.yaml
33-
curl -sSLf "$template" > $tmp_yaml || exit 1
42+
curl -sSLf "$template" > "$tmp_yaml" || exit 1
3443
template=$tmp_yaml
3544
;;
3645
*)
@@ -47,27 +56,85 @@ runs:
4756
arm64) arch=aarch64 ;;
4857
esac
4958
50-
# extract digest and location from template using arch
51-
digest="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].digest // \"\"" "$template")"
52-
location="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].location // \"\"" "$template")"
53-
test -n "$location" || exit 1
54-
55-
# path to cache
56-
if command -v sha256sum > /dev/null; then
57-
sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
58-
elif command -v shasum > /dev/null; then
59-
sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
59+
# extract digest, location and size by parsing template using arch
60+
readonly yq_filter="
61+
[
62+
.images | map(select(.arch == \"${arch}\")) | [.[0,1].location, .[0,1].digest],
63+
.containerd|[.system or .user],
64+
.containerd.archives | map(select(.arch == \"${arch}\")) | [.[0].location, .[0].digest]
65+
]|flatten|.[]
66+
"
67+
if [[ $detect_containerd == "true" ]]; then
68+
parsed=$(LIMA_HOME=$(mktemp -d) limactl validate "$template" --fill 2>/dev/null | yq eval "${yq_filter}")
6069
else
61-
echo "sha256sum or shasum not found" >&2
70+
parsed=$(yq eval "${yq_filter}" "$template")
71+
fi
72+
# macOS earlier than 15.0 uses bash 3.2.57, which does not support readarray -t
73+
# readarray -t arr <<<"$parsed"
74+
while IFS= read -r line; do arr+=("$line"); done <<<"${parsed}"
75+
readonly locations=("${arr[@]:0:2}") digests=("${arr[@]:2:2}")
76+
readonly containerd="${arr[4]}" containerd_location="${arr[5]}" containerd_digest="${arr[6]}"
77+
for ((i = 0; i < ${#locations[@]}; i++)); do
78+
[[ ${locations[i]} != "null" ]] || continue
79+
http_code=$(curl -sIL -w "%{http_code}" "${locations[i]}" -o /dev/null)
80+
if [[ ${http_code} -eq 200 ]]; then
81+
location=${locations[i]}
82+
digest=${digests[i]}
83+
break
84+
fi
85+
done
86+
if [[ -z ${location} ]]; then
87+
echo "Failed to get the image location for ${template}" >&2
6288
exit 1
6389
fi
64-
echo "path=.download/by-url-sha256/$sha256" >> "$GITHUB_OUTPUT"
90+
91+
function location_to_sha256() {
92+
local location=$1
93+
if command -v sha256sum > /dev/null; then
94+
sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
95+
elif command -v shasum > /dev/null; then
96+
sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
97+
else
98+
echo "sha256sum or shasum not found" >&2
99+
exit 1
100+
fi
101+
echo "$sha256"
102+
}
103+
104+
function location_to_cache_path() {
105+
local location=$1
106+
sha256=$(location_to_sha256 "$location") && echo ".download/by-url-sha256/$sha256"
107+
}
108+
109+
# path to cache
110+
image_cache_path=$(location_to_cache_path "$location")
111+
echo "path=$image_cache_path" >> "$GITHUB_OUTPUT"
65112
66113
# key for cache
67-
key="${digest:+image-$digest}"
68-
# fallback to os and hash of template file if digest not found
69-
key="${key:-${{ runner.os }}-${{ hashFiles(inputs.template) }}}"
114+
image_basename=$(basename "$location")
115+
if [[ "$digest" != "null" ]]; then
116+
key="image:$image_basename-$digest"
117+
else
118+
# use sha256 of location as key if digest is not available
119+
key="image:$image_basename-url-sha256:$(location_to_sha256 "$location")"
120+
fi
70121
echo "key=$key" >> "$GITHUB_OUTPUT"
122+
123+
# containerd path and key for cache
124+
if [[ $containerd == "true" && "$containerd_location" != "null" ]]; then
125+
containerd_basename=$(basename "$containerd_location")
126+
if [[ ${containerd_digest} != "null" ]]; then
127+
containerd_key="containerd:$containerd_basename-$containerd_digest"
128+
else
129+
containerd_key="containerd:$containerd_basename-url-sha256:$(sha256 "$containerd_location")"
130+
fi
131+
echo "containerd-key=$containerd_key" >> "$GITHUB_OUTPUT"
132+
containerd_cache_path=$(location_to_cache_path "$containerd_location")
133+
echo "containerd-path=$containerd_cache_path" >> "$GITHUB_OUTPUT"
134+
else
135+
echo "containerd-key=" >> "$GITHUB_OUTPUT"
136+
echo "containerd-path=" >> "$GITHUB_OUTPUT"
137+
fi
71138
shell: bash
72139

73140
- name: "Cache ${{ steps.cache-params-from-template.outputs.path }}"
@@ -78,11 +145,19 @@ runs:
78145
key: ${{ steps.cache-params-from-template.outputs.key }}
79146
enableCrossOsArchive: true
80147

148+
- name: "Cache ${{ steps.cache-params-from-template.outputs.containerd-key }}"
149+
if: ${{ steps.cache-params-from-template.outputs.containerd-key != '' }}
150+
uses: actions/cache@v4
151+
with:
152+
path: ${{ steps.cache-params-from-template.outputs.containerd-path }}
153+
key: ${{ steps.cache-params-from-template.outputs.containerd-key }}
154+
enableCrossOsArchive: true
155+
81156
- name: "Create symbolic link named ${{ steps.detect-platform.outputs.download-dir }} pointing to .download"
82157
run: |
83158
set -eux
84159
[ -d .download ] || mkdir -p .download
85160
path_to_cache=${{ steps.detect-platform.outputs.download-dir }}
86-
mkdir -p $(dirname $path_to_cache)
87-
ln -sfn $PWD/.download $path_to_cache
161+
mkdir -p "$(dirname "$path_to_cache")"
162+
ln -sfn "$PWD/.download" "$path_to_cache"
88163
shell: bash

‎.github/workflows/test.yml

+17-16
Original file line numberDiff line numberDiff line change
@@ -157,16 +157,16 @@ jobs:
157157
- uses: actions/setup-go@v5
158158
with:
159159
go-version: 1.22.x
160-
- name: Cache image used by default.yaml
161-
uses: ./.github/actions/setup_cache_for_template
162-
with:
163-
template: templates/default.yaml
164160
- name: Unit tests
165161
run: go test -v ./...
166162
- name: Make
167163
run: make
168164
- name: Install
169165
run: make install
166+
- name: Cache image used by default.yaml
167+
uses: ./.github/actions/setup_cache_for_template
168+
with:
169+
template: templates/default.yaml
170170
- name: Validate templates
171171
run: find -L templates -name '*.yaml' | xargs limactl validate
172172
- name: Install test dependencies
@@ -225,6 +225,10 @@ jobs:
225225
- uses: actions/setup-go@v5
226226
with:
227227
go-version: 1.22.x
228+
- name: Make
229+
run: make
230+
- name: Install
231+
run: sudo make install
228232
- name: normalize template path
229233
id: normalize_template_path
230234
# `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
@@ -233,10 +237,6 @@ jobs:
233237
uses: ./.github/actions/setup_cache_for_template
234238
with:
235239
template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
236-
- name: Make
237-
run: make
238-
- name: Install
239-
run: sudo make install
240240
- name: Install test dependencies
241241
run: |
242242
sudo apt-get update
@@ -325,14 +325,14 @@ jobs:
325325
- uses: actions/setup-go@v5
326326
with:
327327
go-version: 1.22.x
328-
- name: Cache image used by vmnet.yaml
329-
uses: ./.github/actions/setup_cache_for_template
330-
with:
331-
template: templates/vmnet.yaml
332328
- name: Make
333329
run: make
334330
- name: Install
335331
run: make install
332+
- name: Cache image used by vmnet.yaml
333+
uses: ./.github/actions/setup_cache_for_template
334+
with:
335+
template: templates/vmnet.yaml
336336
- name: Install test dependencies
337337
run: brew install qemu bash coreutils iperf3
338338
- name: Install socket_vmnet
@@ -379,6 +379,7 @@ jobs:
379379
uses: ./.github/actions/setup_cache_for_template
380380
with:
381381
template: https://raw.githubusercontent.com/lima-vm/lima/${{ matrix.oldver }}/examples/ubuntu-lts.yaml
382+
detect-containerd: "false"
382383
- name: Install test dependencies
383384
run: brew install qemu bash coreutils
384385
- name: Test
@@ -408,6 +409,10 @@ jobs:
408409
- uses: actions/setup-go@v5
409410
with:
410411
go-version: 1.22.x
412+
- name: Make
413+
run: make
414+
- name: Install
415+
run: make install
411416
- name: normalize template path
412417
id: normalize_template_path
413418
# `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
@@ -416,10 +421,6 @@ jobs:
416421
uses: ./.github/actions/setup_cache_for_template
417422
with:
418423
template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
419-
- name: Make
420-
run: make
421-
- name: Install
422-
run: make install
423424
- name: Install test dependencies
424425
run: brew install bash coreutils jq
425426
- name: Uninstall qemu

0 commit comments

Comments
 (0)