Skip to content

Commit cb0f1d3

Browse files
AkihiroSudaAkihiroSuda
authored
Merge pull request #2794 from norio-nomura/update-template-centos-stream.sh
Add `hack/update-template-centos-stream.sh`
2 parents c3578ed + 68b8d42 commit cb0f1d3

File tree

2 files changed

+273
-0
lines changed

2 files changed

+273
-0
lines changed

‎hack/update-template-centos-stream.sh

+271
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,271 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu -o pipefail
4+
5+
# Functions in this script assume error handling with 'set -e'.
6+
# To ensure 'set -e' works correctly:
7+
# - Use 'set +e' before assignments and '$(set -e; <function>)' to capture output without exiting on errors.
8+
# - Avoid calling functions directly in conditions to prevent disabling 'set -e'.
9+
# - Use 'shopt -s inherit_errexit' (Bash 4.4+) to avoid repeated 'set -e' in all '$(...)'.
10+
shopt -s inherit_errexit || error_exit "inherit_errexit not supported. Please use bash 4.4 or later."
11+
12+
function centos_print_help() {
13+
cat <<HELP
14+
$(basename "${BASH_SOURCE[0]}"): Update the CentOS Stream image location in the specified templates
15+
16+
Usage:
17+
$(basename "${BASH_SOURCE[0]}") [--version <version>] <template.yaml>...
18+
19+
Description:
20+
This script updates the CentOS Stream image location in the specified templates.
21+
If the image location in the template contains a release date in the URL, the script replaces it with the latest available date.
22+
23+
Image location basename format: CentOS[Stream-GenericCloud-<version>-[latest|<date>.0].<arch>.qcow2
24+
25+
Published CentOS Stream image information is fetched from the following URLs:
26+
27+
https://cloud.centos.org/centos/<major version>-stream/<arch>/images/
28+
29+
To parsing html, this script requires 'htmlq' or 'pup' command.
30+
The downloaded files will be cached in the Lima cache directory.
31+
32+
Examples:
33+
Update the CentOS Stream image location in templates/**.yaml:
34+
$ $(basename "${BASH_SOURCE[0]}") templates/**.yaml
35+
36+
Update the CentOS Stream image location in ~/.lima/centos/lima.yaml:
37+
$ $(basename "${BASH_SOURCE[0]}") ~/.lima/centos/lima.yaml
38+
$ limactl factory-reset centos
39+
40+
Update the CentOS Stream image location to 9-Stream in ~/.lima/centos/lima.yaml:
41+
$ $(basename "${BASH_SOURCE[0]}") --version 9-stream ~/.lima/centos/lima.yaml
42+
$ limactl factory-reset centos
43+
44+
Flags:
45+
--version <version> Use the specified version. The version must be 8 or later.
46+
-h, --help Print this help message
47+
HELP
48+
}
49+
50+
# print the URL spec for the given location
51+
function centos_url_spec_from_location() {
52+
local location=$1 jq_filter url_spec
53+
jq_filter='capture(
54+
"^https://cloud\\.centos\\.org/centos/(?<path_version>\\d+)-stream/(?<path_arch>[^/]+)/images/" +
55+
"CentOS-Stream-(?<target_vendor>.*)-(?<version>\\d+(\\.[.\\d]+)?)-" +
56+
"(latest|(?<date_and_ci_job_id>\\d{8}\\.\\d+))\\.(?<arch>[^.]+).(?<file_extension>.*)$"
57+
;"x")'
58+
url_spec=$(jq -e -r "${jq_filter}" <<<"\"${location}\"")
59+
jq -e '.path_version == .version' <<<"${url_spec}" >/dev/null ||
60+
error_exit "Validation failed: .path_version != .version: ${location}"
61+
jq -e '.path_arch == .arch' <<<"${url_spec}" >/dev/null ||
62+
error_exit "Validation failed: .path_arch != .arch: ${location}"
63+
echo "${url_spec}"
64+
}
65+
66+
readonly centos_jq_filter_directory='"https://cloud.centos.org/centos/\(.version)-stream/\(.path_arch)/images/"'
67+
readonly centos_jq_filter_filename='"CentOS-Stream-\(.target_vendor)-\(.version)-\(.date_and_ci_job_id // "latest").\(.arch).\(.file_extension)"'
68+
69+
# print the location for the given URL spec
70+
function centos_location_from_url_spec() {
71+
local -r url_spec=$1
72+
jq -e -r "${centos_jq_filter_directory} + ${centos_jq_filter_filename}" <<<"${url_spec}" ||
73+
error_exit "Failed to get the location for ${url_spec}"
74+
}
75+
76+
function centos_image_directory_from_url_spec() {
77+
local -r url_spec=$1
78+
jq -e -r "${centos_jq_filter_directory}" <<<"${url_spec}" ||
79+
error_exit "Failed to get the image directory for ${url_spec}"
80+
}
81+
82+
function centos_image_filename_from_url_spec() {
83+
local -r url_spec=$1
84+
jq -e -r "${centos_jq_filter_filename}" <<<"${url_spec}" ||
85+
error_exit "Failed to get the image filename for ${url_spec}"
86+
}
87+
88+
#
89+
function centos_latest_image_entry_for_url_spec() {
90+
local url_spec=$1 version arch image_directory downloaded_page links_in_page latest_info
91+
version=$(jq -r '.version' <<<"${url_spec}")
92+
arch=$(jq -r '.arch' <<<"${url_spec}")
93+
image_directory=$(centos_image_directory_from_url_spec "${url_spec}")
94+
downloaded_page=$(download_to_cache "${image_directory}")
95+
if command -v htmlq >/dev/null; then
96+
links_in_page=$(htmlq 'td.indexcolname a' --attribute href <"${downloaded_page}")
97+
elif command -v pup >/dev/null; then
98+
links_in_page=$(pup 'td[class=indexcolname] a attr{href}' <"${downloaded_page}")
99+
else
100+
error_exit "Please install 'htmlq' or 'pup' to list images from https://cloud.centos.org/centos/${version}/${arch}/images/"
101+
fi
102+
latest_info=$(jq -e -Rrs --argjson spec "${url_spec}" '
103+
[
104+
split("\n").[] |
105+
capture(
106+
"^CentOS-Stream-\($spec.target_vendor)-\($spec.version)-(?<date_and_ci_job_id>\\d{8}\\.\\d+)\\.\($spec.arch)\\.\($spec.file_extension)$"
107+
;"x"
108+
)
109+
] | sort_by(.date_and_ci_job_id) | last
110+
' <<<"${links_in_page}")
111+
[[ -n ${latest_info} ]] || return
112+
local newer_url_spec location sha256sum_location downloaded_sha256sum filename digest
113+
newer_url_spec=$(jq -e -r ". + ${latest_info}" <<<"${url_spec}")
114+
location=$(centos_location_from_url_spec "${newer_url_spec}")
115+
sha256sum_location="${location}.SHA256SUM"
116+
downloaded_sha256sum=$(download_to_cache "${sha256sum_location}")
117+
filename=$(centos_image_filename_from_url_spec "${newer_url_spec}")
118+
digest="sha256:$(awk "/SHA256 \(${filename}\) =/{print \$4}" "${downloaded_sha256sum}")"
119+
[[ -n ${digest} ]] || error_exit "Failed to get the SHA256 digest for ${filename}"
120+
json_vars location arch digest
121+
}
122+
123+
function centos_cache_key_for_image_kernel() {
124+
local location=$1 overriding=${3:-"{}"} url_spec
125+
url_spec=$(centos_url_spec_from_location "${location}" | jq -r ". + ${overriding}")
126+
jq -r '["centos", .version, .target_vendor,
127+
if .date_and_ci_job_id then "timestamped" else "latest" end,
128+
.arch, .file_extension] | join(":")' <<<"${url_spec}"
129+
}
130+
131+
function centos_image_entry_for_image_kernel() {
132+
local location=$1 kernel_is_not_supported=$2 overriding=${3:-"{}"} url_spec image_entry=''
133+
[[ ${kernel_is_not_supported} == "null" ]] || echo "Updating kernel information is not supported on CentOS Stream" >&2
134+
url_spec=$(centos_url_spec_from_location "${location}" | jq -r ". + ${overriding}")
135+
if jq -e '.date_and_ci_job_id' <<<"${url_spec}" >/dev/null; then
136+
image_entry=$(centos_latest_image_entry_for_url_spec "${url_spec}")
137+
else
138+
image_entry=$(
139+
# shellcheck disable=SC2030
140+
location=$(centos_location_from_url_spec "${url_spec}")
141+
location=$(validate_url_without_redirect "${location}")
142+
arch=$(jq -r '.path_arch' <<<"${url_spec}")
143+
json_vars location arch
144+
)
145+
fi
146+
# shellcheck disable=SC2031
147+
if [[ -z ${image_entry} ]]; then
148+
error_exit "Failed to get the ${url_spec} image location for ${location}"
149+
elif jq -e ".location == \"${location}\"" <<<"${image_entry}" >/dev/null; then
150+
echo "Image location is up-to-date: ${location}" >&2
151+
else
152+
echo "${image_entry}"
153+
fi
154+
}
155+
156+
# check if the script is executed or sourced
157+
# shellcheck disable=SC1091
158+
if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then
159+
scriptdir=$(dirname "${BASH_SOURCE[0]}")
160+
# shellcheck source=./cache-common-inc.sh
161+
. "${scriptdir}/cache-common-inc.sh"
162+
163+
if ! command -v htmlq >/dev/null && ! command -v pup >/dev/null; then
164+
error_exit "Please install 'htmlq' or 'pup' to list images from https://cloud.centos.org/centos/<version>/<arch>/images/"
165+
fi
166+
# shellcheck source=/dev/null # avoid shellcheck hangs on source looping
167+
. "${scriptdir}/update-template.sh"
168+
else
169+
# this script is sourced
170+
if ! command -v htmlq >/dev/null && ! command -v pup >/dev/null; then
171+
echo "Please install 'htmlq' or 'pup' to list images from https://cloud.centos.org/centos/<version>/<arch>/images/" >&2
172+
elif [[ -v SUPPORTED_DISTRIBUTIONS ]]; then
173+
SUPPORTED_DISTRIBUTIONS+=("centos")
174+
else
175+
declare -a SUPPORTED_DISTRIBUTIONS=("centos")
176+
fi
177+
return 0
178+
fi
179+
180+
declare -a templates=()
181+
declare overriding="{}"
182+
while [[ $# -gt 0 ]]; do
183+
case "$1" in
184+
-h | --help)
185+
centos_print_help
186+
exit 0
187+
;;
188+
-d | --debug) set -x ;;
189+
--version)
190+
if [[ -n $2 && $2 != -* ]]; then
191+
overriding=$(
192+
version="${2%%-*}"
193+
[[ ${version} -ge 8 ]] || error_exit "CentOS Stream version must be 8 or later"
194+
json_vars version <<<"${overriding}"
195+
)
196+
shift
197+
else
198+
error_exit "--version requires a value"
199+
fi
200+
;;
201+
--version=*)
202+
overriding=$(
203+
version="${1#*=}"
204+
version="${version%%-*}"
205+
[[ ${version} -ge 8 ]] || error_exit "CentOS Stream version must be 8 or later"
206+
json_vars version <<<"${overriding}"
207+
)
208+
;;
209+
*.yaml) templates+=("$1") ;;
210+
*)
211+
error_exit "Unknown argument: $1"
212+
;;
213+
esac
214+
shift
215+
[[ -z ${overriding} ]] && overriding="{}"
216+
done
217+
218+
if [[ ${#templates[@]} -eq 0 ]]; then
219+
centos_print_help
220+
exit 0
221+
fi
222+
223+
declare -A image_entry_cache=()
224+
225+
for template in "${templates[@]}"; do
226+
echo "Processing ${template}"
227+
# 1. extract location by parsing template using arch
228+
yq_filter="
229+
.images[] | [.location, .kernel.location, .kernel.cmdline] | @tsv
230+
"
231+
parsed=$(yq eval "${yq_filter}" "${template}")
232+
233+
# 3. get the image location
234+
arr=()
235+
while IFS= read -r line; do arr+=("${line}"); done <<<"${parsed}"
236+
locations=("${arr[@]}")
237+
for ((index = 0; index < ${#locations[@]}; index++)); do
238+
[[ ${locations[index]} != "null" ]] || continue
239+
set -e
240+
IFS=$'\t' read -r location kernel_location kernel_cmdline <<<"${locations[index]}"
241+
set +e # Disable 'set -e' to avoid exiting on error for the next assignment.
242+
cache_key=$(
243+
set -e # Enable 'set -e' for the next command.
244+
centos_cache_key_for_image_kernel "${location}" "${kernel_location}" "${overriding}"
245+
) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition.
246+
# shellcheck disable=2181
247+
[[ $? -eq 0 ]] || continue
248+
image_entry=$(
249+
set -e # Enable 'set -e' for the next command.
250+
if [[ -v image_entry_cache[${cache_key}] ]]; then
251+
echo "${image_entry_cache[${cache_key}]}"
252+
else
253+
centos_image_entry_for_image_kernel "${location}" "${kernel_location}" "${overriding}"
254+
fi
255+
) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition.
256+
# shellcheck disable=2181
257+
[[ $? -eq 0 ]] || continue
258+
set -e
259+
image_entry_cache[${cache_key}]="${image_entry}"
260+
if [[ -n ${image_entry} ]]; then
261+
[[ ${kernel_cmdline} != "null" ]] &&
262+
jq -e 'has("kernel")' <<<"${image_entry}" >/dev/null &&
263+
image_entry=$(jq ".kernel.cmdline = \"${kernel_cmdline}\"" <<<"${image_entry}")
264+
echo "${image_entry}" | jq
265+
limactl edit --log-level error --set "
266+
.images[${index}] = ${image_entry}|
267+
(.images[${index}] | ..) style = \"double\"
268+
" "${template}"
269+
fi
270+
done
271+
done

‎hack/update-template.sh

+2
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,8 @@ if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then
150150
. "${scriptdir}/update-template-debian.sh"
151151
# shellcheck source=./update-template-archlinux.sh
152152
. "${scriptdir}/update-template-archlinux.sh"
153+
# shellcheck source=./update-template-centos-stream.sh
154+
. "${scriptdir}/update-template-centos-stream.sh"
153155
else
154156
# this script is sourced
155157
return 0

0 commit comments

Comments
 (0)