Skip to content

Commit a3d0d7e

Browse files
norio-nomuranorio-nomura
committed
setup_cache_for_template: Improve caching mechanism for images and nerdctl archives
- Change cache key to `url-sha256:$sha256` for caching images without a digest - Include image basename in the cache key - Use `limactl validate --fill` to retrieve nerdctl archive info and set cache if needed - `test.yml`: Change cache configuration to run after `make install` Signed-off-by: Norio Nomura <[email protected]>
1 parent 402ba1c commit a3d0d7e

File tree

2 files changed

+107
-34
lines changed

2 files changed

+107
-34
lines changed

‎.github/actions/setup_cache_for_template/action.yml

+90-18
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ inputs:
77
template:
88
description: template yaml file
99
required: true
10+
detect-containerd:
11+
description: detect containerd usage from template by using limactl validate
12+
required: false
13+
default: 'true'
1014
runs:
1115
using: "composite"
1216
steps:
@@ -27,10 +31,15 @@ runs:
2731
set -eux
2832
arch="${{ inputs.arch }}"
2933
template="${{ inputs.template }}"
34+
detect_containerd="${{ inputs.detect-containerd }}"
35+
if [[ $detect_containerd == "true" ]] && ! command -v limactl &>/dev/null; then
36+
echo "containerd detection is disabled because limactl is not found" >&2
37+
detect_containerd="false"
38+
fi
3039
case "$template" in
3140
https://*)
3241
tmp_yaml=$(mktemp -d)/template.yaml
33-
curl -sSLf "$template" > $tmp_yaml || exit 1
42+
curl -sSLf "$template" > "$tmp_yaml" || exit 1
3443
template=$tmp_yaml
3544
;;
3645
*)
@@ -47,27 +56,82 @@ runs:
4756
arm64) arch=aarch64 ;;
4857
esac
4958
50-
# extract digest and location from template using arch
51-
digest="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].digest // \"\"" "$template")"
52-
location="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].location // \"\"" "$template")"
53-
test -n "$location" || exit 1
54-
55-
# path to cache
56-
if command -v sha256sum > /dev/null; then
57-
sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
58-
elif command -v shasum > /dev/null; then
59-
sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
59+
# extract digest, location and size by parsing template using arch
60+
readonly yq_filter="
61+
[
62+
.images | map(select(.arch == \"${arch}\")) | [.[0,1].location, .[0,1].digest],
63+
.containerd|[.system or .user],
64+
.containerd.archives | map(select(.arch == \"${arch}\")) | [.[0].location, .[0].digest]
65+
]|flatten|.[]
66+
"
67+
if [[ $detect_containerd == "true" ]]; then
68+
parsed=$(LIMA_HOME=$(mktemp -d) limactl validate "$template" --fill 2>/dev/null | yq eval "${yq_filter}")
6069
else
61-
echo "sha256sum or shasum not found" >&2
70+
parsed=$(yq eval "${yq_filter}" "$template")
71+
fi
72+
readarray -t arr <<<"$parsed"
73+
readonly locations=("${arr[@]:0:2}") digests=("${arr[@]:2:2}")
74+
readonly containerd="${arr[4]}" containerd_location="${arr[5]}" containerd_digest="${arr[6]}"
75+
for ((i = 0; i < ${#locations[@]}; i++)); do
76+
[[ ${locations[i]} != "null" ]] || continue
77+
http_code=$(curl -sIL -w "%{http_code}" "${locations[i]}" -o /dev/null)
78+
if [[ ${http_code} -eq 200 ]]; then
79+
location=${locations[i]}
80+
digest=${digests[i]}
81+
break
82+
fi
83+
done
84+
if [[ -z ${location} ]]; then
85+
echo "Failed to get the image location for ${template}" >&2
6286
exit 1
6387
fi
64-
echo "path=.download/by-url-sha256/$sha256" >> "$GITHUB_OUTPUT"
88+
89+
function location_to_sha256() {
90+
local location=$1
91+
if command -v sha256sum > /dev/null; then
92+
sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
93+
elif command -v shasum > /dev/null; then
94+
sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
95+
else
96+
echo "sha256sum or shasum not found" >&2
97+
exit 1
98+
fi
99+
echo "$sha256"
100+
}
101+
102+
function location_to_cache_path() {
103+
sha256=$(location_to_sha256 "$location") && echo ".download/by-url-sha256/$sha256"
104+
}
105+
106+
# path to cache
107+
image_cache_path=$(location_to_cache_path "$location")
108+
echo "path=$image_cache_path" >> "$GITHUB_OUTPUT"
65109
66110
# key for cache
67-
key="${digest:+image-$digest}"
68-
# fallback to os and hash of template file if digest not found
69-
key="${key:-${{ runner.os }}-${{ hashFiles(inputs.template) }}}"
111+
image_basename=$(basename "$location")
112+
if [[ "$digest" != "null" ]]; then
113+
key="image:$image_basename-$digest"
114+
else
115+
# use sha256 of location as key if digest is not available
116+
key="image:$image_basename-url-sha256:$(sha256 "$location")"
117+
fi
70118
echo "key=$key" >> "$GITHUB_OUTPUT"
119+
120+
# containerd path and key for cache
121+
if [[ $containerd == "true" && "$containerd_location" != "null" ]]; then
122+
containerd_basename=$(basename "$containerd_location")
123+
if [[ ${containerd_digest} != "null" ]]; then
124+
containerd_key="containerd:$containerd_basename-$containerd_digest"
125+
else
126+
containerd_key="containerd:$containerd_basename-url-sha256:$(sha256 "$containerd_location")"
127+
fi
128+
echo "containerd-key=$containerd_key" >> "$GITHUB_OUTPUT"
129+
containerd_cache_path=$(location_to_cache_path "$containerd_location")
130+
echo "containerd-path=$containerd_cache_path" >> "$GITHUB_OUTPUT"
131+
else
132+
echo "containerd-key=" >> "$GITHUB_OUTPUT"
133+
echo "containerd-path=" >> "$GITHUB_OUTPUT"
134+
fi
71135
shell: bash
72136

73137
- name: "Cache ${{ steps.cache-params-from-template.outputs.path }}"
@@ -78,11 +142,19 @@ runs:
78142
key: ${{ steps.cache-params-from-template.outputs.key }}
79143
enableCrossOsArchive: true
80144

145+
- name: "Cache ${{ steps.cache-params-from-template.outputs.containerd-key }}"
146+
if: ${{ steps.cache-params-from-template.outputs.containerd-key != '' }}
147+
uses: actions/cache@v4
148+
with:
149+
path: ${{ steps.cache-params-from-template.outputs.containerd-path }}
150+
key: ${{ steps.cache-params-from-template.outputs.containerd-key }}
151+
enableCrossOsArchive: true
152+
81153
- name: "Create symbolic link named ${{ steps.detect-platform.outputs.download-dir }} pointing to .download"
82154
run: |
83155
set -eux
84156
[ -d .download ] || mkdir -p .download
85157
path_to_cache=${{ steps.detect-platform.outputs.download-dir }}
86-
mkdir -p $(dirname $path_to_cache)
87-
ln -sfn $PWD/.download $path_to_cache
158+
mkdir -p "$(dirname "$path_to_cache")"
159+
ln -sfn "$PWD/.download" "$path_to_cache"
88160
shell: bash

‎.github/workflows/test.yml

+17-16
Original file line numberDiff line numberDiff line change
@@ -157,16 +157,16 @@ jobs:
157157
- uses: actions/setup-go@v5
158158
with:
159159
go-version: 1.22.x
160-
- name: Cache image used by default.yaml
161-
uses: ./.github/actions/setup_cache_for_template
162-
with:
163-
template: templates/default.yaml
164160
- name: Unit tests
165161
run: go test -v ./...
166162
- name: Make
167163
run: make
168164
- name: Install
169165
run: make install
166+
- name: Cache image used by default.yaml
167+
uses: ./.github/actions/setup_cache_for_template
168+
with:
169+
template: templates/default.yaml
170170
- name: Validate templates
171171
run: find -L templates -name '*.yaml' | xargs limactl validate
172172
- name: Install test dependencies
@@ -225,6 +225,10 @@ jobs:
225225
- uses: actions/setup-go@v5
226226
with:
227227
go-version: 1.22.x
228+
- name: Make
229+
run: make
230+
- name: Install
231+
run: sudo make install
228232
- name: normalize template path
229233
id: normalize_template_path
230234
# `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
@@ -233,10 +237,6 @@ jobs:
233237
uses: ./.github/actions/setup_cache_for_template
234238
with:
235239
template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
236-
- name: Make
237-
run: make
238-
- name: Install
239-
run: sudo make install
240240
- name: Install test dependencies
241241
run: |
242242
sudo apt-get update
@@ -325,14 +325,14 @@ jobs:
325325
- uses: actions/setup-go@v5
326326
with:
327327
go-version: 1.22.x
328-
- name: Cache image used by vmnet.yaml
329-
uses: ./.github/actions/setup_cache_for_template
330-
with:
331-
template: templates/vmnet.yaml
332328
- name: Make
333329
run: make
334330
- name: Install
335331
run: make install
332+
- name: Cache image used by vmnet.yaml
333+
uses: ./.github/actions/setup_cache_for_template
334+
with:
335+
template: templates/vmnet.yaml
336336
- name: Install test dependencies
337337
run: brew install qemu bash coreutils iperf3
338338
- name: Install socket_vmnet
@@ -379,6 +379,7 @@ jobs:
379379
uses: ./.github/actions/setup_cache_for_template
380380
with:
381381
template: https://raw.githubusercontent.com/lima-vm/lima/${{ matrix.oldver }}/examples/ubuntu-lts.yaml
382+
detect-containerd: "false"
382383
- name: Install test dependencies
383384
run: brew install qemu bash coreutils
384385
- name: Test
@@ -408,6 +409,10 @@ jobs:
408409
- uses: actions/setup-go@v5
409410
with:
410411
go-version: 1.22.x
412+
- name: Make
413+
run: make
414+
- name: Install
415+
run: make install
411416
- name: normalize template path
412417
id: normalize_template_path
413418
# `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
@@ -416,10 +421,6 @@ jobs:
416421
uses: ./.github/actions/setup_cache_for_template
417422
with:
418423
template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
419-
- name: Make
420-
run: make
421-
- name: Install
422-
run: make install
423424
- name: Install test dependencies
424425
run: brew install bash coreutils jq
425426
- name: Uninstall qemu

0 commit comments

Comments
 (0)