ci: Add test cases for CoCo image pulling without forked containerd

.ci/ci_job_flags.sh

@@ -127,6 +127,7 @@ case "${CI_JOB}" in
 			export AA_KBC="offline_fs_kbc"
 			if [[ "${CI_JOB}" =~ K8S ]]; then
 				export KUBERNETES=yes
+				export SNAPSHOTTER="nydus"
 			fi
 			;;
 	esac
@@ -191,6 +192,7 @@ case "${CI_JOB}" in
 		if grep -q 'prot_virt=1' /proc/cmdline && grep -Eq '^facilities.* 158 .*' /proc/cpuinfo; then
 			export TEE_TYPE="se"
 		fi
+		export SNAPSHOTTER="nydus"
 	fi
 
 	if [[ "${CI_JOB}" =~ CLOUD_HYPERVISOR ]]; then

.ci/install_cri_containerd.sh

@@ -17,6 +17,8 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Flag to do tasks for CI
 CI=${CI:-""}
 
+SNAPSHOTTER=${SNAPSHOTTER:-""}
+
 # shellcheck source=./lib.sh
 source "${script_dir}/lib.sh"
 
@@ -81,17 +83,42 @@ install_from_static_tarball() {
 	sudo tar -xvf "${tarball_name}" -C /
 }
 
+install_vanilla_from_static_tarball() {
+	echo "Trying to install containerd from static tarball"
+	local tarball_url="https://github.com/containerd/containerd/releases/download"
+	local containerd_tarball_version="v1.7.0"
+	local containerd_version=${containerd_tarball_version#v}
+	local tarball_name="containerd-${containerd_version}-${CONTAINERD_OS}-${CONTAINERD_ARCH}.tar.gz"
+	local url="${tarball_url}/${containerd_tarball_version}/${tarball_name}"
+
+	echo "Download tarball from ${url}"
+	if ! curl -OL -f "${url}"; then
+		echo "Failed to download tarball from ${url}"
+		return 1
+	fi
+
+	tmp_dir=$(mktemp -d -t install-vanilla-containerd-tmp.XXXXXXXXXX)
+	sudo tar -xvf "${tarball_name}" -C $tmp_dir/
+	sudo install -D -m 755 "$tmp_dir/bin/containerd" "/usr/local/bin/containerd-vanilla"
+
+	systemctl status containerd
+}
+
 install_cri-tools() {
-crictl_repo=$(get_version "externals.critools.url")
-crictl_version=$(get_version "externals.critools.version")
-crictl_tag_prefix="v"
+	crictl_repo=$(get_version "externals.critools.url")
+	crictl_version=$(get_version "externals.critools.version")
+	crictl_tag_prefix="v"
 
-crictl_url="${crictl_repo}/releases/download/v${crictl_version}/crictl-${crictl_tag_prefix}${crictl_version}-linux-$(${script_dir}/kata-arch.sh -g).tar.gz"
-curl -Ls "$crictl_url" | sudo tar xfz - -C /usr/local/bin
+	crictl_url="${crictl_repo}/releases/download/v${crictl_version}/crictl-${crictl_tag_prefix}${crictl_version}-linux-$(${script_dir}/kata-arch.sh -g).tar.gz"
+	curl -Ls "$crictl_url" | sudo tar xfz - -C /usr/local/bin
 }
 
 install_from_static_tarball || install_from_source
 
+if [ "${SNAPSHOTTER}" == "nydus" ]; then
+	install_vanilla_from_static_tarball
+fi
+
 install_cri-tools
 
 sudo systemctl daemon-reload

.ci/install_nydus_snapshotter.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+cidir=$(dirname "$0")
+source "${cidir}/lib.sh"
+
+target_dir="/opt/kata"
+
+nydus_snapshotter_repo=${nydus_snapshotter_repo:-"github.com/containerd/nydus-snapshotter"}
+nydus_snapshotter_repo_git="https://${nydus_snapshotter_repo}.git"
+nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.12.0"}
+nydus_snapshotter_repo_dir="${GOPATH}/src/${nydus_snapshotter_repo}"
+nydus_snapshotter_binary_target_dir="$target_dir/bin"
+nydus_snapshotter_config_target_dir="$target_dir/share/nydus-snapshotter"
+
+nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"}
+nydus_version=${nydus_version:-"v2.2.3"}
+
+arch="$(uname -m)"
+
+clone_nydus_snapshotter_repo() {
+    add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}"
+
+    if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then
+        sudo mkdir -p "${nydus_snapshotter_repo_dir}"
+        sudo git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true
+        pushd "${nydus_snapshotter_repo_dir}"
+        sudo git checkout "${nydus_snapshotter_version}"
+        popd
+    fi
+}
+
+build_nydus_snapshotter() {
+    pushd "${nydus_snapshotter_repo_dir}"
+    if [ "$arch" = "s390x" ]; then
+        export GOARCH=$arch
+    fi
+    sudo -E PATH=$PATH make
+
+    sudo install -D -m 755 "bin/containerd-nydus-grpc" "$nydus_snapshotter_binary_target_dir/containerd-nydus-grpc"
+    sudo install -D -m 755 "bin/nydus-overlayfs" "$nydus_snapshotter_binary_target_dir/nydus-overlayfs"
+    if [ ! -f "/usr/local/bin/nydus-overlayfs" ]; then
+        echo " /usr/local/bin/nydus-overlayfs exists, now we will replace it."
+        sudo cp "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs"
+    fi
+    sudo rm -rf "$nydus_snapshotter_repo_dir/bin"
+    popd >/dev/null
+}
+
+download_nydus_snapshotter_config() {
+    tmp_dir=$(mktemp -d -t install-nydus-snapshotter-config-tmp.XXXXXXXXXX)
+    sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$tmp_dir/config-coco-guest-pulling.toml"
+    sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$tmp_dir/config-coco-host-sharing.toml"
+    sudo install -D -m 644 "$tmp_dir/config-coco-guest-pulling.toml" "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
+    sudo install -D -m 644 "$tmp_dir/config-coco-host-sharing.toml" "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
+
+}
+
+download_nydus_from_tarball() {
+    if [ "$arch" = "s390x" ]; then
+        echo "Skip to download nydus for $arch, it doesn't work for $arch now."
+        return
+    fi
+    local goarch="$(${cidir}/kata-arch.sh --golang)"
+    local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-$goarch.tgz"
+    echo "Download tarball from ${tarball_url}"
+    tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX)
+    sudo curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1
+    sudo install -D -m 755 "$tmp_dir/nydus-image" "/usr/local/bin/"
+}
+
+download_nydus_from_tarball
+clone_nydus_snapshotter_repo
+build_nydus_snapshotter
+download_nydus_snapshotter_config
+echo "install nydus-snapshotter successful"

.ci/lib.sh

@@ -391,6 +391,18 @@ cleanup_network_interface() {
 	[ "$CNI" != "" ] && info "$CNI doesn't clean up"
 }
 
+cleanup_nydus_snapshotter_dependencies() {
+	if [ -f "/usr/local/bin/nydus-overlayfs" ]; then
+		rm -f "/usr/local/bin/nydus-overlayfs"
+	fi
+	if [ -f "/usr/local/bin/nydus-image" ]; then
+		rm -f "/usr/local/bin/nydus-image"
+	fi
+	if [ -f "/usr/local/bin/containerd-vanilla" ]; then
+		rm -f "/usr/local/bin/containerd-vanilla"
+	fi
+}
+
 gen_clean_arch() {
 	# For metrics CI we are removing unnecessary steps like
 	# removing packages, removing CRI-O, etc mainly because
@@ -412,6 +424,9 @@ gen_clean_arch() {
 		fi
 	fi
 
+	info "remove nydus snapshotter dependencies"
+	cleanup_nydus_snapshotter_dependencies
+
 	info "remove containers started by ctr"
 	clean_env_ctr
 

.ci/setup.sh

@@ -24,6 +24,7 @@ KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
 KUBERNETES="${KUBERNETES:-yes}"
 TEST_CGROUPSV2="${TEST_CGROUPSV2:-false}"
 USE_DEVMAPPER="${USE_DEVMAPPER:-false}"
+SNAPSHOTTER="${SNAPSHOTTER:-}"
 
 setup_distro_env() {
 	local setup_type="$1"
@@ -135,6 +136,11 @@ install_extra_tools() {
 		fi
 	fi
 
+	if [ "${SNAPSHOTTER}" == "nydus" ]; then
+		info "Install nydus-snapshotter"
+		bash -f "${cidir}/install_nydus_snapshotter.sh"
+	fi
+
 	echo "Install CNI plugins"
 	bash -f "${cidir}/install_cni_plugins.sh"
 }

Makefile

@@ -104,7 +104,7 @@ cc-containerd:
 # Run the Confidential Containers tests for kubernetes.
 cc-kubernetes:
 	bash -f .ci/install_bats.sh
-	K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \
+	K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \
 	bash integration/kubernetes/run_kubernetes_tests.sh
 
 # Run the Confidential Containers AMD SEV specific tests.

integration/confidential/lib.sh

@@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
 source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
 FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
 SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
+NYDUS_SNAPSHOTTER_BINARY="/opt/kata/bin/containerd-nydus-grpc"
+NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-host-sharing.toml"
+NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-guest-pulling.toml"
+NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
 
 # Toggle between true and false the service_offload configuration of
 # the Kata agent.
@@ -180,6 +184,13 @@ disable_full_debug() {
 	sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH"
 }
 
+restart_containerd() {
+	sudo systemctl restart containerd
+	if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
+		die "containerd seems not operational after restarted"
+	fi
+}
+
 # Configure containerd for confidential containers. Among other things, it ensures
 # the CRI handler is configured to deal with confidential container.
 #
@@ -198,15 +209,15 @@ configure_cc_containerd() {
 	# installed via operator it will assume containerd is in right state
 	# already.
 	[ "${TESTS_CONFIGURE_CC_CONTAINERD:-yes}" == "yes" ] || return 0
+	sudo iptables -w -P FORWARD ACCEPT
 
 	# Even if we are not saving the original file it is a good idea to
 	# restart containerd because it might be in an inconsistent state here.
 	sudo systemctl stop containerd
-	sleep 5
+	sleep 10
 	[ -n "$saved_containerd_conf_file" ] && \
 		sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
-	sudo systemctl start containerd
-	waitForProcess 30 5 "sudo crictl info >/dev/null"
+	restart_containerd
 
 	# Ensure the cc CRI handler is set.
 	local cri_handler=$(sudo crictl info | \
@@ -223,11 +234,6 @@ configure_cc_containerd() {
 			  sudo tee -a "$containerd_conf_file"
 	fi
 
-	sudo systemctl restart containerd
-	if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
-		die "containerd seems not operational after reconfigured"
-	fi
-	sudo iptables -w -P FORWARD ACCEPT
 }
 
 #
@@ -445,3 +451,91 @@ EOF
 EOF
   fi
 }
+
+###############################################################################
+
+# remote-snapshotter
+
+EXPORT_MODE=${EXPORT_MODE:-"image_guest_pull"}
+
+configure_remote_snapshotter() {
+	case "${SNAPSHOTTER}" in
+	"nydus")
+		configure_nydus_snapshotter
+		;;
+	*) ;;
+
+	esac
+}
+
+is_containerd_support_per_runtime_snapshotter() {
+   containerd_version=$(containerd --version | awk '{print $3}')
+   required_version="v1.7.0"
+   printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet  -V
+}
+
+set_vanilla_containerd() {
+	sudo systemctl stop containerd
+	sleep 10
+	sudo mv -f /usr/local/bin/containerd /usr/local/bin/containerd-coco
+	sudo cp -f /usr/local/bin/containerd-vanilla /usr/local/bin/containerd
+	echo "vanilla containerd version: $(containerd --version | awk '{print $3}')"
+	restart_containerd
+}
+
+unset_vanilla_containerd() {
+	sudo systemctl stop containerd
+	sleep 10
+	sudo rm -f /usr/local/bin/containerd 
+	sudo mv -f /usr/local/bin/containerd-coco /usr/local/bin/containerd
+	echo "coco containerd version: $(containerd --version | awk '{print $3}')"
+	restart_containerd
+}
+
+configure_containerd_for_nydus_snapshotter() {
+	set_vanilla_containerd
+	local containerd_config="$1"
+	snapshotter_socket="/run/containerd-nydus/containerd-nydus-grpc.sock"
+	proxy_config="  [proxy_plugins.$SNAPSHOTTER]\n    type = \"snapshot\"\n    address = \"${snapshotter_socket}\""
+
+	if grep -q "\[proxy_plugins\]" "$containerd_config"; then
+		sudo sed -i '/\[proxy_plugins\]/a\'"$proxy_config" "$containerd_config"
+	else
+		sudo echo -e "[proxy_plugins]" >>"$containerd_config"
+		sudo echo -e "$proxy_config" >>"$containerd_config"
+	fi
+
+	sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config"
+	sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config"
+}
+
+kill_nydus_snapshotter_process() {
+	echo "Kill nydus snapshotter"
+	bin="containerd-nydus-grpc"
+	sudo kill -9 $(pidof $bin) || true
+	sudo rm -rf "/var/lib/containerd-nydus" || true
+}
+
+remove_test_image() {
+	local test_image="$1"
+	crictl rmi "$1"
+	pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
+	crictl rmi "$pause_name"
+}
+
+restart_nydus_snapshotter() {
+	kill_nydus_snapshotter_process || true
+	echo "Restart nydus snapshotter"
+	sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 &
+}
+
+configure_nydus_snapshotter() {
+	echo "Configure nydus snapshotter"
+	if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
+		NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
+	else
+		NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
+		sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
+	fi
+	restart_nydus_snapshotter
+}

integration/containerd/confidential/tests_common.sh

@@ -26,7 +26,7 @@ setup_common() {
 	echo "Prepare containerd for Confidential Container"
 	SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
 	configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
-
+	restart_containerd
 	# Note: ensure that intructions changing the kernel parameters are
 	# executed *after* saving the original list.
 	saved_kernel_params=$(get_kernel_params)

integration/kubernetes/confidential/agent_image.bats

@@ -31,7 +31,9 @@ RUNTIMECLASS="${RUNTIMECLASS:-kata}"
 test_tag="[cc][agent][kubernetes][containerd]"
 
 setup() {
-	setup_common
+	setup_containerd
+    restart_containerd
+    reconfigure_kata
 }
 
 @test "$test_tag Test can launch pod with measured boot enabled" {