Start writing user documentation for Image finding.

garloff · garloff · commit e40df25e3009 · 2025-11-26T15:37:28.000+01:00
Just to show what to do if we don't have standardized image names any
longer. Python and Shell (CLI) covered, opentofu is TBD.

Signed-off-by: Kurt Garloff &lt;kurt@garloff.de&gt;
diff --git a/user-docs/usage-hints/find-image/find_img.py b/user-docs/usage-hints/find-image/find_img.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+#
+# find-img.py
+#
+# Searches for an image with distribution and version
+#
+# (c) Kurt Garloff <s7n@garloff.de>, 7/2025
+# SPDX-License-Identifier: MIT
+"This module finds the a vanilla distribution image"
+
+import os
+import sys
+import openstack
+# import logging
+
+
+def warn(log, msg):
+    "warn output"
+    if log:
+        log.warn(msg)
+    else:
+        print(f"WARN: {msg}", file=sys.stderr)
+
+
+def debug(log, msg):
+    "debug output"
+    if log:
+        log.debug(msg)
+    else:
+        print(f"DEBUG: {msg}", file=sys.stderr)
+
+
+def img_sort_heuristic(images, distro, version, purpose):
+    """Sort list to prefer old names"""
+    # Do sorting magic (could be omitted)
+    newlist = []
+    distro = distro.lower()
+    version = version.lower()
+    purpose = purpose.lower()
+    # 0th: Exact match old SCS naming scheme ("Ubuntu 24.04 Minimal")
+    for img in images:
+        newel = (img.id, img.name)
+        if img.name.lower() == f"{distro} {version} {purpose}":
+            newlist.append(newel)
+        elif img.name.lower() == f"{distro} {purpose} {version}":
+            newlist.append(newel)
+    # 1st: Exact match old SCS naming scheme ("Ubuntu 24.04")
+    for img in images:
+        newel = (img.id, img.name)
+        if img.name.lower() == f"{distro} {version}":
+            newlist.append(newel)
+    # 2nd: Fuzzy match old SCS naming scheme ("Ubuntu 24.04*")
+    for img in images:
+        newel = (img.id, img.name)
+        if img.name.lower().startswith(f"{distro} {version}") and newel not in newlist:
+            newlist.append(newel)
+    # 3rd: Even more fuzzy match old SCS naming scheme ("Ubuntu*24.04")
+    for img in images:
+        newel = (img.id, img.name)
+        if img.name.lower().startswith(f"{distro}") and img.name.lower().endswith(f"{version}") \
+                and newel not in newlist:
+            newlist.append(newel)
+    # 4th: Rest
+    for img in images:
+        newel = (img.id, img.name)
+        if newel not in newlist:
+            newlist.append(newel)
+    return newlist
+
+
+def find_image(conn, distro, version, purpose="generic", strict=False, log=None):
+    """Return a sorted list of ID,Name pairs that contain the wanted image.
+       Empty list indicates no image has been found. The list is sorted such
+       that (on SCS-compliant clouds), it will very likely contain the most
+       vanilla, most recent image as first element.
+       If strict is set, multiple matches are not allowed.
+    """
+    ldistro = distro.lower()
+    # FIXME: The image.images() method only passes selected filters
+    images = [x for x in conn.image.images(os_distro=ldistro, os_version=version,
+                                           sort="name:desc,created_at:desc", visibility="public")
+              if x.properties.get("os_purpose") == purpose]
+    if len(images) == 0:
+        warn(log, f"No image found with os_distro={ldistro} os_version={version} os_purpose={purpose}")
+        # images = list(conn.image.images(os_distro=ldistro, os_version=version,
+        #                                 sort="name:desc,created_at:desc"))
+        images = [x for x in conn.image.images(os_distro=ldistro, os_version=version,
+                                               sort="name:desc,created_at:desc")
+                  if "os_purpose" not in x.properties]
+    if len(images) == 0:
+        warn(log, f"No image found with os_distro={ldistro} os_version={version} without os_purpose")
+        return []
+    # Now comes sorting magic for best backwards compatibility
+    if len(images) > 1:
+        debug(log, f"Several {purpose} images found with os_distro={ldistro} os_version={version}")
+        if (strict):
+            return []
+        return img_sort_heuristic(images, distro, version, purpose)
+    return [(img.id, img.name) for img in images]
+
+
+def usage():
+    "Usage hints (CLI)"
+    print("Usage: find-img.sh [-s] DISTRO VERSION [PURPOSE]", file=sys.stderr)
+    print("Returns all images matching, latest first, purpose defaulting to generic", file=sys.stderr)
+    print("[-s] sets strict mode where only one match is allowed.", file=sys.stderr)
+    print("You need to have OS_CLOUD set when running this", file=sys.stderr)
+    sys.exit(1)
+
+
+def main(argv):
+    "Main entry for CLI"
+    if len(argv) < 3:
+        usage()
+    try:
+        conn = openstack.connect(cloud=os.environ["OS_CLOUD"])
+    except openstack.exceptions.ConfigException:
+        print(f"No valid entry for cloud {os.environ['OS_CLOUD']}", file=sys.stderr)
+        usage()
+    except KeyError:
+        print("OS_CLOUD environment not configured", file=sys.stderr)
+        usage()
+    conn.authorize()
+    purpose = "generic"
+    strict = False
+    if argv[1] == "-s":
+        argv = argv[1:]
+        strict = True
+    if len(argv) > 3:
+        purpose = argv[3]
+    images = find_image(conn, argv[1], argv[2], purpose, strict)
+    for img in images:
+        print(f"{img[0]} {img[1]}")
+    return len(images) == 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))
diff --git a/user-docs/usage-hints/find-image/find_img.sh b/user-docs/usage-hints/find-image/find_img.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Find Image by properties
+#
+# (c) Kurt Garloff <s7n@garloff.de>, 7/2025
+# SPDX-License-Identifier: MIT
+
+usage()
+{
+	echo "Usage: find-img distro version [purpose]"
+	echo "Returns all images matching, latest first, purpose defaults to generic"
+	echo "If some images have the wanted purpose, only those will be shown"
+}
+
+get_images_raw()
+{
+	# global OS_RESP
+	DIST=$(echo "$1" | tr A-Z a-z)
+	VERS="$2"
+	#VERS=$(echo "$2" | tr A-Z a-z)
+	shift; shift
+	#echo "DEBUG: openstack image list --property os_distro=$DIST --property os_version=$VERS $@ -f value -c ID -c Name --sort created_at:desc"
+	OS_RESP=$(openstack image list --property os_distro="$DIST" --property os_version="$VERS" $@ -f value -c ID -c Name --sort name:desc,created_at:desc)
+}
+
+
+img_sort_heuristic()
+{
+	# Acts on global OS_RESP
+	# FIXME: We could do all sorts of advanced heuristics here, looking at the name etc.
+	# We only do one thing here: Sort the image that matches the old naming scheme first.
+	# distro version purpose
+	local NEW_RESP0=$(echo "$OS_RESP" | grep -i "^[0-9a-f\-]* $1 $2 $3\$")
+	# distro version purpose with extras appended
+	local NEW_RESP1=$(echo "$OS_RESP" | grep -i "^[0-9a-f\-]* $1 $2 $3" | grep -iv "^[0-9a-f\-]* $1 $2 $3\$")
+	# distro purpose version
+	local NEW_RESP2=$(echo "$OS_RESP" | grep -i "^[0-9a-f\-]* $1 $3 $2\$")
+	# distro purpose version with extras appended
+	local NEW_RESP3=$(echo "$OS_RESP" | grep -i "^[0-9a-f\-]* $1 $3 $2" | grep -iv "^[0-9a-f\-]* $1 $3 $2\$")
+	# distro version
+	local NEW_RESP4=$(echo "$OS_RESP" | grep -i "^[0-9a-f\-]* $1 $2\$")
+	# distro version with extras (but not purpose)
+	local NEW_RESP5=$(echo "$OS_RESP" | grep -i "^[0-9a-f\-]* $1 $2" | grep -iv "^[0-9a-f\-]* $1 $2\$" | grep -iv "^[0-9a-f\-]* $1 $2 $3")
+	# distro extra version (but extra != purpose)
+	local NEW_RESP6=$(echo "$OS_RESP" | grep -i "^[0-9a-f\-]* $1 .*$2\$" | grep -iv "^[0-9a-f\-]* $1 $3 $2\$" | grep -iv "$1 $2\$")
+	OS_RESP=$(echo -e "$NEW_RESP0\n$NEW_RESP1\n$NEW_RESP2\n$NEW_RESP3\n$NEW_RESP4\n$NEW_RESP5\n$NEW_RESP6" | sed '/^$/d')
+}
+
+get_images()
+{
+	PURPOSE=${3:-generic}
+	get_images_raw "$1" "$2" --property os_purpose=$PURPOSE
+	if test -z "$OS_RESP"; then 
+		echo "WARN: No image found with os_distro=$1 os_version=$2 os_purpose=$PURPOSE" 1>&2
+		# We're screwed as we can not filter for the absence of os_purpose with CLI
+		# We could loop and do an image show and then flter out, but that's very slow
+		get_images_raw "$1" "$2" # --property os_purpose=
+		# FIXME: We need to filter out images with os_purpose property set
+		NEW_RESP=""
+		while read ID Name; do
+			PROPS=$(openstack image show $ID -f value -c properties)
+			if test $? != 0; then continue; fi
+			if echo "$PROPS" | grep os_purpose >/dev/null 2>&1; then continue; fi
+			NEW_RESP=$(echo -en "$NEW_RESP\n$ID $Name")
+		done < <(echo "$OS_RESP")
+		OS_RESP=$(echo "$NEW_RESP" | sed '/^$/d')
+	fi
+	NR_IMG=$(echo "$OS_RESP" | sed '/^$/d' | wc -l)
+	if test "$NR_IMG" = "0"; then echo "ERROR: No image found with os_distro=$1 os_version=$2" 1>&2; return 1
+	elif test "$NR_IMG" = "1"; then return 0
+	else
+		echo "DEBUG: Several $PURPOSE images matching os_distro=$1 os_version=$2" 1>&2;
+		if test -n "$STRICT"; then return 1; fi
+		img_sort_heuristic "$1" "$2" "$PURPOSE"
+		return 0
+	fi
+}
+
+if test -z "$OS_CLOUD" -a -z "$OS_AUTH_URL"; then
+	echo "You need to configure clouds.yaml/secure.yaml and set OS_CLOUD" 1>&2
+	exit 2
+fi
+if test -z "$1"; then
+	usage
+	exit 1
+fi
+
+get_images "$@"
+RC=$?
+echo "$OS_RESP"
+(exit $RC)
diff --git a/user-docs/usage-hints/find-image/index.md b/user-docs/usage-hints/find-image/index.md
@@ -0,0 +1,111 @@
+---
+layout: post
+title: 'Locating provider-managed images'
+author:
+  - 'Kurt Garloff'
+avatar:
+  - 'kgarloff.jpg'
+---
+
+## Purpose
+
+Many providers provide public images that they maintain for user convenience.
+Maintenance means that they regularly update it to include the latest bug- and
+security fixes. The exact policy is transparent from the image metadata as
+specified in SCS standard [scs-0102](https://docs.scs.community/standards/iaas/scs-0102).
+A few images have to be managed this way by the provider according to
+[scs-0104](https://docs.scs.community/standards/iaas/scs-0104).
+Previously (with [scs-0102-v1](https://docs.scs.community/standards/scs-0102-v1-image-metadata))
+the image could be referenced by a standard name to always get the current
+image whereas a reference by UUID would result in an unchanged image (until
+it is removed according to the provider's policy that is transparent from
+the metadata).
+
+Some providers prefer to use different image names. We intend to allow this with
+[scs-0102-v2](https://docs.scs.community/standards/scs-0102-v2-image-metadata).
+This however means that identifying the most recent "Ubuntu 24.04" image on
+an SCS-compatible IaaS cloud becomes a bit harder in a portable way.
+This article describes how to do this.
+
+## The new `os_purpose` property
+
+While we suggest to rename or better to hide old images, there can still legitimately
+be several variants of images, e.g. minimal variants or Kubernetes node images etc.
+These must not be confused with the standard general purpose images. To avoid
+confusion, we have introduce a new `os_purpose` (recommended in v1.1 of scs-0102
+and mandatory in v2) field, that can be set to `generic`, `minimal`, `k8snode`,
+`gpu`, `network`, or `custom` in v2.
+To now find the latest general purpose Ubuntu Noble Numbat 24.04 image, one can search the
+image catalog for `os_distro=ubuntu`, `os_version=24.04`, and `os_purpose=generic`.
+This is straightforward if all SCS clouds already comply to the new metadata standard
+and only have one matching image.
+It's a bit more complex in case we have to deal with a mixture of old and new ...
+
+## Identifying the right image using python (openstack-SDK)
+
+To find the Ubuntu 24.04 generic image, we would just do
+
+```python
+    images = [x for x in conn.image.images(os_distro=distro, os_version=version,
+                                           sort="name:desc,created_at:desc")
+              if x.properties.get("os_purpose") == purpose]
+```
+
+where `conn` is a connection to your OpenStack project and `distro`, `version` and
+`purpose` have been set to the lowercase strings you are looking for.
+
+Three notes:
+
+- We use a list comprehension to filter for `os_purpose` because `os_purpose`
+  is not one of the hardcoded properties that the SDK knows unlike `os_distro`
+  and `os_version`.
+- We can add additional filtering such as `visibility="public"` if we just want
+  to look for public images.
+- We sort the list, so in case we have several matches, we want the images grouped
+  by image name and within the same name have the latest images first. This would
+  typically find the latest image both in the case where a provider renames old
+  images "Ubuntu 24.04" to "Ubuntu 24.04 timestamp" or fails to rename them.
+  (The latter would not be compliant with scs-0102.)
+
+It gets a bit harder when you want SCS clouds that comply to the old v1 standard
+and do not yet have the `os_purpose` field set. Above call then returns an empty
+list. We then would fall back to look for images that match `os_distro` and
+`os_version`, but have no `os_purpose` property.
+
+```python
+        images = [x for x in conn.image.images(os_distro=distro, os_version=version,
+                                               sort="name:desc,created_at:desc")
+                  if "os_purpose" not in x.properties]
+```
+
+We have to expect several matches here and need some heuristic to find the
+right image, preferrably the one matching the old naming convention.
+
+Full code that does this is available in [find_img.py](find_img.py).
+Feel free to copy, I deliberately put this under MIT license.
+
+## Identifying the image with OpenStack CLI
+
+Unlike with Python, we can pass the `os_purpose` field just like the other
+properties.
+
+```bash
+ openstack image list --property os_distro="$DIST" --property os_version="$VERS" --property os_purpose="$PURP" -f value -c ID -c Name --sort name:desc,created_at:desc
+```
+
+where `OS_CLOUD` environment has been configured to access your cloud project and
+`DIST`, `VERS` and `PURP` are set to the lowercased image properties you
+are looking for. An additional filter `--public` parameter could be passed to only
+list public images. See above python comment for the sorting rationale.
+
+Dealing with old SCS clouds (not yet implementing v2 of scs-0102) is harder
+with shell code. The reason is that we can not pass a flag to `openstack
+image list` that would tell it to restrict results to records without an
+`os_purpose` property. So this requires looping over the images and filtering
+out all images with `os_purpose` (but not matching our request).
+
+Full code that does this is available in [find_img.sh](find_img.sh).
+
+## Terraform / opentofu
+
+TBW
