From f8cdfd21d0e3f9c25cfae1014d550b8e22c81da1 Mon Sep 17 00:00:00 2001
From: ChengyuZhu6 <chengyu.zhu@intel.com>
Date: Wed, 6 Sep 2023 11:14:23 +0800
Subject: [PATCH 1/3] ci: Add test cases for CoCo image pulling without forked
 containerd

Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo.
1)image sharing on the host without dm-verity.
2)image sharing on the host with dm-verity.
3)image pulling in the guest with nydus-snapshotter.

Fixes #5763

Depends-on: github.com/kata-containers/kata-containers#7676

Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>
---
 .ci/install_nydus_snapshotter.sh              |  90 +++++++++++++++
 Makefile                                      |   2 +-
 integration/confidential/lib.sh               | 108 ++++++++++++++++--
 .../containerd/confidential/tests_common.sh   |   2 +-
 .../kubernetes/confidential/agent_image.bats  |   4 +-
 .../confidential/agent_image_encrypted.bats   |  12 +-
 .../confidential/fixtures/pod-config.yaml.in  |   2 +-
 .../image_pulling_with_snapshotter.bats       |  71 ++++++++++++
 integration/kubernetes/confidential/lib.sh    |   3 +-
 .../confidential/sealed_secret.bats           |  10 +-
 integration/kubernetes/confidential/sev.bats  |   4 +-
 integration/kubernetes/confidential/snp.bats  |   2 +
 .../kubernetes/confidential/tests_common.sh   |  26 ++++-
 13 files changed, 303 insertions(+), 33 deletions(-)
 create mode 100644 .ci/install_nydus_snapshotter.sh
 create mode 100644 integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

diff --git a/.ci/install_nydus_snapshotter.sh b/.ci/install_nydus_snapshotter.sh
new file mode 100644
index 000000000..5e2da6c6f
--- /dev/null
+++ b/.ci/install_nydus_snapshotter.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+cidir=$(dirname "$0")
+source "${cidir}/lib.sh"
+
+target_dir="/opt/confidential-containers"
+
+nydus_snapshotter_repo=${nydus_snapshotter_repo:-"github.com/containerd/nydus-snapshotter"}
+nydus_snapshotter_repo_git="https://${nydus_snapshotter_repo}.git"
+nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.12.0"}
+nydus_snapshotter_repo_dir="${GOPATH}/src/${nydus_snapshotter_repo}"
+nydus_snapshotter_binary_target_dir="$target_dir/bin"
+nydus_snapshotter_config_target_dir="$target_dir/share/nydus-snapshotter"
+
+nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"}
+nydus_version=${nydus_version:-"v2.2.3"}
+
+arch="$(uname -m)"
+
+clone_nydus_snapshotter_repo() {
+    add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}"
+
+    if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then
+        sudo mkdir -p "${nydus_snapshotter_repo_dir}"
+        git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true
+        pushd "${nydus_snapshotter_repo_dir}"
+        git checkout "${nydus_snapshotter_version}"
+        popd
+    fi
+}
+
+build_nydus_snapshotter() {
+    pushd "${nydus_snapshotter_repo_dir}"
+    if [ "$arch" = "s390x" ]; then
+        export GOARCH=$arch
+    fi
+    sudo -E PATH=$PATH make
+
+    if [ ! -d "$nydus_snapshotter_binary_target_dir" ]; then
+        sudo mkdir -p $nydus_snapshotter_binary_target_dir
+    fi
+    sudo install -D -m 755 "bin/containerd-nydus-grpc" "$nydus_snapshotter_binary_target_dir/containerd-nydus-grpc"
+    sudo install -D -m 755 "bin/nydus-overlayfs" "$nydus_snapshotter_binary_target_dir/nydus-overlayfs"
+    if [ ! -f "/usr/local/bin/nydus-overlayfs" ]; then
+        echo " /usr/local/bin/nydus-overlayfs exists, now we will replace it."
+        sudo cp -f "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs"
+    fi
+    sudo rm -rf "$nydus_snapshotter_repo_dir/bin"
+    popd >/dev/null
+}
+
+download_nydus_snapshotter_config() {
+    if [ ! -d "$nydus_snapshotter_config_target_dir" ]; then
+        mkdir -p "$nydus_snapshotter_config_target_dir"
+    fi
+    sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
+    sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
+    sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
+    sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
+
+}
+
+download_nydus_from_tarball() {
+    if [ "$arch" = "s390x" ]; then
+        echo "Skip to download nydus for $arch, it doesn't work for $arch now."
+        return
+    fi
+    local goarch="$(${cidir}/kata-arch.sh --golang)"
+    local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-$goarch.tgz"
+    echo "Download tarball from ${tarball_url}"
+    tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX)
+    curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1
+    sudo install -D -m 755 "$tmp_dir/nydus-image" "/usr/local/bin/"
+}
+
+download_nydus_from_tarball
+clone_nydus_snapshotter_repo
+build_nydus_snapshotter
+download_nydus_snapshotter_config
+echo "install nydus-snapshotter successful"
diff --git a/Makefile b/Makefile
index 48b892219..b2a5c07e8 100644
--- a/Makefile
+++ b/Makefile
@@ -104,7 +104,7 @@ cc-containerd:
 # Run the Confidential Containers tests for kubernetes.
 cc-kubernetes:
 	bash -f .ci/install_bats.sh
-	K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \
+	K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \
 	bash integration/kubernetes/run_kubernetes_tests.sh
 
 # Run the Confidential Containers AMD SEV specific tests.
diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh
index 6e0873d05..9ec87e02c 100644
--- a/integration/confidential/lib.sh
+++ b/integration/confidential/lib.sh
@@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
 source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
 FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
 SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
+NYDUS_SNAPSHOTTER_BINARY="/opt/kata/bin/containerd-nydus-grpc"
+NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-host-sharing.toml"
+NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-guest-pulling.toml"
+NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
 
 # Toggle between true and false the service_offload configuration of
 # the Kata agent.
@@ -180,6 +184,13 @@ disable_full_debug() {
 	sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH"
 }
 
+restart_containerd() {
+	sudo systemctl restart containerd
+	if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
+		die "containerd seems not operational after restarted"
+	fi
+}
+
 # Configure containerd for confidential containers. Among other things, it ensures
 # the CRI handler is configured to deal with confidential container.
 #
@@ -198,6 +209,7 @@ configure_cc_containerd() {
 	# installed via operator it will assume containerd is in right state
 	# already.
 	[ "${TESTS_CONFIGURE_CC_CONTAINERD:-yes}" == "yes" ] || return 0
+	sudo iptables -w -P FORWARD ACCEPT
 
 	# Even if we are not saving the original file it is a good idea to
 	# restart containerd because it might be in an inconsistent state here.
@@ -205,8 +217,7 @@ configure_cc_containerd() {
 	sleep 5
 	[ -n "$saved_containerd_conf_file" ] && \
 		sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
-	sudo systemctl start containerd
-	waitForProcess 30 5 "sudo crictl info >/dev/null"
+	restart_containerd
 
 	# Ensure the cc CRI handler is set.
 	local cri_handler=$(sudo crictl info | \
@@ -223,11 +234,6 @@ configure_cc_containerd() {
 			  sudo tee -a "$containerd_conf_file"
 	fi
 
-	sudo systemctl restart containerd
-	if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
-		die "containerd seems not operational after reconfigured"
-	fi
-	sudo iptables -w -P FORWARD ACCEPT
 }
 
 #
@@ -445,3 +451,91 @@ EOF
 EOF
   fi
 }
+
+###############################################################################
+
+# remote-snapshotter
+
+EXPORT_MODE=${EXPORT_MODE:-"image_guest_pull"}
+
+configure_remote_snapshotter() {
+	case "${SNAPSHOTTER}" in
+	"nydus")
+		configure_nydus_snapshotter
+		;;
+	*) ;;
+
+	esac
+}
+
+is_containerd_support_per_runtime_snapshotter() {
+   containerd_version=$(containerd --version | awk '{print $3}')
+   required_version="v1.7.0"
+   printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet  -V
+}
+
+set_vanilla_containerd() {
+	sudo systemctl stop containerd
+	sleep 5
+	sudo mv /usr/local/bin/containerd /usr/local/bin/containerd-coco
+	sudo cp /usr/local/bin/containerd-vanilla /usr/local/bin/containerd
+	echo "vanilla containerd version: $(containerd --version | awk '{print $3}')"
+	restart_containerd
+}
+
+unset_vanilla_containerd() {
+	sudo systemctl stop containerd
+	sleep 5
+	sudo rm -f /usr/local/bin/containerd 
+	sudo mv /usr/local/bin/containerd-coco /usr/local/bin/containerd
+	echo "coco containerd version: $(containerd --version | awk '{print $3}')"
+	restart_containerd
+}
+
+configure_containerd_for_nydus_snapshotter() {
+	set_vanilla_containerd
+	local containerd_config="$1"
+	snapshotter_socket="/run/containerd-nydus/containerd-nydus-grpc.sock"
+	proxy_config="  [proxy_plugins.$SNAPSHOTTER]\n    type = \"snapshot\"\n    address = \"${snapshotter_socket}\""
+
+	if grep -q "\[proxy_plugins\]" "$containerd_config"; then
+		sudo sed -i '/\[proxy_plugins\]/a\'"$proxy_config" "$containerd_config"
+	else
+		sudo echo -e "[proxy_plugins]" >>"$containerd_config"
+		sudo echo -e "$proxy_config" >>"$containerd_config"
+	fi
+
+	sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config"
+	sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config"
+}
+
+kill_nydus_snapshotter_process() {
+	echo "Kill nydus snapshotter"
+	bin="containerd-nydus-grpc"
+	sudo kill -9 $(pidof $bin) || true
+	sudo rm -rf "/var/lib/containerd-nydus" || true
+}
+
+remove_test_image() {
+	local test_image="$1"
+	crictl rmi "$1"
+	pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
+	crictl rmi "$pause_name"
+}
+
+restart_nydus_snapshotter() {
+	kill_nydus_snapshotter_process || true
+	echo "Restart nydus snapshotter"
+	sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 &
+}
+
+configure_nydus_snapshotter() {
+	echo "Configure nydus snapshotter"
+	if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
+		NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
+	else
+		NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
+		sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
+	fi
+	restart_nydus_snapshotter
+}
\ No newline at end of file
diff --git a/integration/containerd/confidential/tests_common.sh b/integration/containerd/confidential/tests_common.sh
index c44f08bf6..3d3c5362f 100644
--- a/integration/containerd/confidential/tests_common.sh
+++ b/integration/containerd/confidential/tests_common.sh
@@ -26,7 +26,7 @@ setup_common() {
 	echo "Prepare containerd for Confidential Container"
 	SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
 	configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
-
+	restart_containerd
 	# Note: ensure that intructions changing the kernel parameters are
 	# executed *after* saving the original list.
 	saved_kernel_params=$(get_kernel_params)
diff --git a/integration/kubernetes/confidential/agent_image.bats b/integration/kubernetes/confidential/agent_image.bats
index de5c815e8..b0bf11320 100644
--- a/integration/kubernetes/confidential/agent_image.bats
+++ b/integration/kubernetes/confidential/agent_image.bats
@@ -31,7 +31,9 @@ RUNTIMECLASS="${RUNTIMECLASS:-kata}"
 test_tag="[cc][agent][kubernetes][containerd]"
 
 setup() {
-	setup_common
+	setup_containerd
+    restart_containerd
+    reconfigure_kata
 }
 
 @test "$test_tag Test can launch pod with measured boot enabled" {
diff --git a/integration/kubernetes/confidential/agent_image_encrypted.bats b/integration/kubernetes/confidential/agent_image_encrypted.bats
index b8d842ed9..febe2b1ca 100644
--- a/integration/kubernetes/confidential/agent_image_encrypted.bats
+++ b/integration/kubernetes/confidential/agent_image_encrypted.bats
@@ -4,9 +4,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-load "${BATS_TEST_DIRNAME}/lib.sh"
-load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
 load "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 # Allow to configure the runtimeClassName on pod configuration.
 RUNTIMECLASS="${RUNTIMECLASS:-kata}"
@@ -25,13 +24,8 @@ setup() {
     SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
     configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
 
-    echo "Reconfigure Kata Containers"
-    switch_image_service_offload on
-    clear_kernel_params
-    add_kernel_params "${original_kernel_params}"
-
-    setup_proxy
-    switch_measured_rootfs_verity_scheme none
+    restart_containerd
+    reconfigure_kata
 }
 
 @test "$test_tag Test can pull an encrypted image inside the guest with decryption key" {
diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
index 3c8e9d3c1..1d0b7def0 100644
--- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
+++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
@@ -5,7 +5,7 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: busybox-cc
+  name: busybox-cc$INDEX
 spec:
   runtimeClassName: $RUNTIMECLASS
   containers:
diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats
new file mode 100644
index 000000000..8c68afdef
--- /dev/null
+++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats
@@ -0,0 +1,71 @@
+#!/usr/bin/env bats
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+tag_suffix=""
+if [ "$(uname -m)" != "x86_64" ]; then
+    tag_suffix="-$(uname -m)"
+fi
+
+# Images used on the tests.
+
+image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
+
+original_kernel_params=$(get_kernel_params)
+# Allow to configure the runtimeClassName on pod configuration.
+RUNTIMECLASS="${RUNTIMECLASS:-kata}"
+test_tag="[cc][agent][kubernetes][containerd]"
+
+setup() {
+    remove_test_image "$image_unsigned_protected" || true
+    setup_containerd
+    restart_containerd
+    configure_containerd_for_nydus_snapshotter "/etc/containerd/config.toml"
+    reconfigure_kata
+    switch_image_service_offload off
+}
+
+@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
+    if [ "$(uname -m)" = "s390x" ]; then
+        skip "test for s390x as nydus-image doesn't currently support this platform"
+    fi
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_protected")"
+        echo $pod_config
+        create_test_pod
+    fi
+}
+
+@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" {
+    if [ "$(uname -m)" = "s390x" ]; then
+        skip "test for s390x as nydus-image doesn't currently support this platform"
+    fi
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_protected")"
+        echo $pod_config
+        create_test_pod
+    fi
+}
+
+@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
+    switch_image_service_offload on
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_protected")"
+        echo $pod_config
+        create_test_pod
+    fi
+}
+
+teardown() {
+    teardown_common
+    remove_test_image "$image_unsigned_protected" || true
+    kill_nydus_snapshotter_process
+    unset_vanilla_containerd
+}
diff --git a/integration/kubernetes/confidential/lib.sh b/integration/kubernetes/confidential/lib.sh
index 9e9d904cc..d85f0ce00 100755
--- a/integration/kubernetes/confidential/lib.sh
+++ b/integration/kubernetes/confidential/lib.sh
@@ -46,7 +46,8 @@ kubernetes_create_cc_pod() {
 	fi
 
 	kubectl apply -f ${config_file}
-	if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
+	pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name')
+	if ! kubectl get pod "$pod_name" &> /dev/null; then
 		echo "Failed to create the pod"
 		return 1
 	fi
diff --git a/integration/kubernetes/confidential/sealed_secret.bats b/integration/kubernetes/confidential/sealed_secret.bats
index 282005547..49995dc62 100644
--- a/integration/kubernetes/confidential/sealed_secret.bats
+++ b/integration/kubernetes/confidential/sealed_secret.bats
@@ -7,6 +7,7 @@
 load "${BATS_TEST_DIRNAME}/lib.sh"
 load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
 load "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 # Allow to configure the runtimeClassName on pod configuration.
 RUNTIMECLASS="${RUNTIMECLASS:-kata}"
@@ -22,13 +23,8 @@ setup() {
     SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
     configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
 
-    echo "Reconfigure Kata Containers"
-    switch_image_service_offload on
-    clear_kernel_params
-    add_kernel_params "${original_kernel_params}"
-
-    setup_proxy
-    switch_measured_rootfs_verity_scheme none
+    restart_containerd
+    reconfigure_kata
 
     kubectl delete secret sealed-secret --ignore-not-found
     # Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault
diff --git a/integration/kubernetes/confidential/sev.bats b/integration/kubernetes/confidential/sev.bats
index 9f515a8bb..0f587cf0c 100644
--- a/integration/kubernetes/confidential/sev.bats
+++ b/integration/kubernetes/confidential/sev.bats
@@ -25,6 +25,7 @@ load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
 load "${TESTS_REPO_DIR}/lib/common.bash"
 load "${TESTS_REPO_DIR}/integration/kubernetes/lib.sh"
 load "${TESTS_REPO_DIR}/integration/kubernetes/confidential/lib.sh"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 # Delete all test services
 k8s_delete_all() {
@@ -55,7 +56,8 @@ setup_file() {
   # Configure CoCo settings in containerd config
   local saved_containerd_conf_file="/etc/containerd/config.toml.$$"
   configure_cc_containerd "${saved_containerd_conf_file}"
-
+  restart_containerd
+  
   # KBS setup and run
   echo "Setting up simple-kbs..."
   simple_kbs_run
diff --git a/integration/kubernetes/confidential/snp.bats b/integration/kubernetes/confidential/snp.bats
index 3f03ebda2..3b7fd586b 100644
--- a/integration/kubernetes/confidential/snp.bats
+++ b/integration/kubernetes/confidential/snp.bats
@@ -21,6 +21,7 @@ export SSH_KEY_FILE
 load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
 load "${TESTS_REPO_DIR}/lib/common.bash"
 load "${TESTS_REPO_DIR}/integration/kubernetes/lib.sh"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 # Delete all test services
 k8s_delete_all() {
@@ -38,6 +39,7 @@ setup_file() {
   # Configure CoCo settings in containerd config
   local saved_containerd_conf_file="/etc/containerd/config.toml.$$"
   configure_cc_containerd "${saved_containerd_conf_file}"
+  restart_containerd
 
   # Pull unencrypted image and retrieve ssh keys
   echo "Pulling unencrypted image and retrieve ssh key..."
diff --git a/integration/kubernetes/confidential/tests_common.sh b/integration/kubernetes/confidential/tests_common.sh
index 177ed450e..68fb8043b 100644
--- a/integration/kubernetes/confidential/tests_common.sh
+++ b/integration/kubernetes/confidential/tests_common.sh
@@ -16,7 +16,6 @@ original_kernel_params=$(get_kernel_params)
 # Global variables exported:
 #	$test_start_date     - test start time.
 #	$pod_config          - path to default pod configuration file.
-#	$original_kernel_params - saved the original list of kernel parameters.
 #
 setup_common() {
 	test_start_date=$(date +"%Y-%m-%d %H:%M:%S")
@@ -25,16 +24,28 @@ setup_common() {
 	pod_id=""
 
 	kubernetes_delete_all_cc_pods_if_any_exists || true
+}
+
+#  Setup containerd for tests.
+#
+setup_containerd() {
+	setup_common
 
 	echo "Prepare containerd for Confidential Container"
 	SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
 	configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
+}
 
+#  Reconfigure Kata for tests.
+#
+# Global variables exported:
+#	$original_kernel_params - saved the original list of kernel parameters.
+#
+reconfigure_kata() {
 	echo "Reconfigure Kata Containers"
 	switch_image_service_offload on
 	clear_kernel_params
 	add_kernel_params "${original_kernel_params}"
-	
 	setup_proxy
 	switch_measured_rootfs_verity_scheme none
 }
@@ -58,6 +69,13 @@ teardown_common() {
 	add_kernel_params "${original_kernel_params}"
 	switch_image_service_offload off
 	disable_full_debug
+	# Restore containerd to pre-test state.
+	if [ -f "$SAVED_CONTAINERD_CONF_FILE" ]; then
+		systemctl stop containerd || true
+		sleep 5
+		mv -f "$SAVED_CONTAINERD_CONF_FILE" "/etc/containerd/config.toml"
+		systemctl start containerd || true
+	fi
 }
 
 
@@ -76,7 +94,6 @@ create_test_pod() {
 		enable_full_debug
 		enable_agent_console
 	fi
-
 	echo "Create the test sandbox"
 	echo "Pod config is: $pod_config"
 	kubernetes_create_cc_pod $pod_config
@@ -97,8 +114,9 @@ create_test_pod() {
 new_pod_config() {
 	local base_config="${FIXTURES_DIR}/pod-config.yaml.in"
 	local image="$1"
+	local index="${2:-}"
 
 	local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
-	IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config"
+	IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS"  INDEX="$index" envsubst < "$base_config" > "$new_config"
 	echo "$new_config"
 }

From b4c9d12a1a9ca482971d3d296b25b9df8cf14adb Mon Sep 17 00:00:00 2001
From: ChengyuZhu6 <chengyu.zhu@intel.com>
Date: Mon, 11 Sep 2023 18:34:23 +0800
Subject: [PATCH 2/3] ci: install nydus snapshotter

Install nydus-snapshotter dependencies.

Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>
---
 .ci/ci_job_flags.sh              |  2 ++
 .ci/install_nydus_snapshotter.sh | 25 ++++++++++---------------
 .ci/lib.sh                       | 12 ++++++++++++
 .ci/setup.sh                     |  6 ++++++
 4 files changed, 30 insertions(+), 15 deletions(-)
 mode change 100644 => 100755 .ci/install_nydus_snapshotter.sh

diff --git a/.ci/ci_job_flags.sh b/.ci/ci_job_flags.sh
index 851215805..be20e23e7 100755
--- a/.ci/ci_job_flags.sh
+++ b/.ci/ci_job_flags.sh
@@ -127,6 +127,7 @@ case "${CI_JOB}" in
 			export AA_KBC="offline_fs_kbc"
 			if [[ "${CI_JOB}" =~ K8S ]]; then
 				export KUBERNETES=yes
+				export SNAPSHOTTER="nydus"
 			fi
 			;;
 	esac
@@ -191,6 +192,7 @@ case "${CI_JOB}" in
 		if grep -q 'prot_virt=1' /proc/cmdline && grep -Eq '^facilities.* 158 .*' /proc/cpuinfo; then
 			export TEE_TYPE="se"
 		fi
+		export SNAPSHOTTER="nydus"
 	fi
 
 	if [[ "${CI_JOB}" =~ CLOUD_HYPERVISOR ]]; then
diff --git a/.ci/install_nydus_snapshotter.sh b/.ci/install_nydus_snapshotter.sh
old mode 100644
new mode 100755
index 5e2da6c6f..0474cf230
--- a/.ci/install_nydus_snapshotter.sh
+++ b/.ci/install_nydus_snapshotter.sh
@@ -13,7 +13,7 @@ set -o errtrace
 cidir=$(dirname "$0")
 source "${cidir}/lib.sh"
 
-target_dir="/opt/confidential-containers"
+target_dir="/opt/kata"
 
 nydus_snapshotter_repo=${nydus_snapshotter_repo:-"github.com/containerd/nydus-snapshotter"}
 nydus_snapshotter_repo_git="https://${nydus_snapshotter_repo}.git"
@@ -32,9 +32,9 @@ clone_nydus_snapshotter_repo() {
 
     if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then
         sudo mkdir -p "${nydus_snapshotter_repo_dir}"
-        git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true
+        sudo git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true
         pushd "${nydus_snapshotter_repo_dir}"
-        git checkout "${nydus_snapshotter_version}"
+        sudo git checkout "${nydus_snapshotter_version}"
         popd
     fi
 }
@@ -46,27 +46,22 @@ build_nydus_snapshotter() {
     fi
     sudo -E PATH=$PATH make
 
-    if [ ! -d "$nydus_snapshotter_binary_target_dir" ]; then
-        sudo mkdir -p $nydus_snapshotter_binary_target_dir
-    fi
     sudo install -D -m 755 "bin/containerd-nydus-grpc" "$nydus_snapshotter_binary_target_dir/containerd-nydus-grpc"
     sudo install -D -m 755 "bin/nydus-overlayfs" "$nydus_snapshotter_binary_target_dir/nydus-overlayfs"
     if [ ! -f "/usr/local/bin/nydus-overlayfs" ]; then
         echo " /usr/local/bin/nydus-overlayfs exists, now we will replace it."
-        sudo cp -f "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs"
+        sudo cp "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs"
     fi
     sudo rm -rf "$nydus_snapshotter_repo_dir/bin"
     popd >/dev/null
 }
 
 download_nydus_snapshotter_config() {
-    if [ ! -d "$nydus_snapshotter_config_target_dir" ]; then
-        mkdir -p "$nydus_snapshotter_config_target_dir"
-    fi
-    sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
-    sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
-    sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
-    sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
+    tmp_dir=$(mktemp -d -t install-nydus-snapshotter-config-tmp.XXXXXXXXXX)
+    sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$tmp_dir/config-coco-guest-pulling.toml"
+    sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$tmp_dir/config-coco-host-sharing.toml"
+    sudo install -D -m 644 "$tmp_dir/config-coco-guest-pulling.toml" "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
+    sudo install -D -m 644 "$tmp_dir/config-coco-host-sharing.toml" "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
 
 }
 
@@ -79,7 +74,7 @@ download_nydus_from_tarball() {
     local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-$goarch.tgz"
     echo "Download tarball from ${tarball_url}"
     tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX)
-    curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1
+    sudo curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1
     sudo install -D -m 755 "$tmp_dir/nydus-image" "/usr/local/bin/"
 }
 
diff --git a/.ci/lib.sh b/.ci/lib.sh
index 6de2dee64..9505aba8f 100755
--- a/.ci/lib.sh
+++ b/.ci/lib.sh
@@ -391,6 +391,15 @@ cleanup_network_interface() {
 	[ "$CNI" != "" ] && info "$CNI doesn't clean up"
 }
 
+cleanup_nydus_snapshotter_dependencies() {
+	if [ -f "/usr/local/bin/nydus-overlayfs" ]; then
+		rm -f "/usr/local/bin/nydus-overlayfs"
+	fi
+	if [ -f "/usr/local/bin/nydus-image" ]; then
+		rm -f "/usr/local/bin/nydus-image"
+	fi
+}
+
 gen_clean_arch() {
 	# For metrics CI we are removing unnecessary steps like
 	# removing packages, removing CRI-O, etc mainly because
@@ -412,6 +421,9 @@ gen_clean_arch() {
 		fi
 	fi
 
+	info "remove nydus snapshotter dependencies"
+	cleanup_nydus_snapshotter_dependencies
+
 	info "remove containers started by ctr"
 	clean_env_ctr
 
diff --git a/.ci/setup.sh b/.ci/setup.sh
index 71cb314db..3e2c67f58 100755
--- a/.ci/setup.sh
+++ b/.ci/setup.sh
@@ -24,6 +24,7 @@ KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
 KUBERNETES="${KUBERNETES:-yes}"
 TEST_CGROUPSV2="${TEST_CGROUPSV2:-false}"
 USE_DEVMAPPER="${USE_DEVMAPPER:-false}"
+SNAPSHOTTER="${SNAPSHOTTER:-}"
 
 setup_distro_env() {
 	local setup_type="$1"
@@ -135,6 +136,11 @@ install_extra_tools() {
 		fi
 	fi
 
+	if [ "${SNAPSHOTTER}" == "nydus" ]; then
+		info "Install nydus-snapshotter"
+		bash -f "${cidir}/install_nydus_snapshotter.sh"
+	fi
+
 	echo "Install CNI plugins"
 	bash -f "${cidir}/install_cni_plugins.sh"
 }

From 31211836e1502f582afde949f1a296373966a40a Mon Sep 17 00:00:00 2001
From: ChengyuZhu6 <chengyu.zhu@intel.com>
Date: Tue, 19 Sep 2023 16:54:23 +0800
Subject: [PATCH 3/3] .ci: add function to install vanilla containerd

add function to install vanilla containerd
Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>
---
 .ci/install_cri_containerd.sh   | 37 ++++++++++++++++++++++++++++-----
 .ci/lib.sh                      |  3 +++
 integration/confidential/lib.sh | 12 +++++------
 3 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/.ci/install_cri_containerd.sh b/.ci/install_cri_containerd.sh
index ad5a11a80..2d586b3f0 100755
--- a/.ci/install_cri_containerd.sh
+++ b/.ci/install_cri_containerd.sh
@@ -17,6 +17,8 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Flag to do tasks for CI
 CI=${CI:-""}
 
+SNAPSHOTTER=${SNAPSHOTTER:-""}
+
 # shellcheck source=./lib.sh
 source "${script_dir}/lib.sh"
 
@@ -81,17 +83,42 @@ install_from_static_tarball() {
 	sudo tar -xvf "${tarball_name}" -C /
 }
 
+install_vanilla_from_static_tarball() {
+	echo "Trying to install containerd from static tarball"
+	local tarball_url="https://github.com/containerd/containerd/releases/download"
+	local containerd_tarball_version="v1.7.0"
+	local containerd_version=${containerd_tarball_version#v}
+	local tarball_name="containerd-${containerd_version}-${CONTAINERD_OS}-${CONTAINERD_ARCH}.tar.gz"
+	local url="${tarball_url}/${containerd_tarball_version}/${tarball_name}"
+
+	echo "Download tarball from ${url}"
+	if ! curl -OL -f "${url}"; then
+		echo "Failed to download tarball from ${url}"
+		return 1
+	fi
+
+	tmp_dir=$(mktemp -d -t install-vanilla-containerd-tmp.XXXXXXXXXX)
+	sudo tar -xvf "${tarball_name}" -C $tmp_dir/
+	sudo install -D -m 755 "$tmp_dir/bin/containerd" "/usr/local/bin/containerd-vanilla"
+
+	systemctl status containerd
+}
+
 install_cri-tools() {
-crictl_repo=$(get_version "externals.critools.url")
-crictl_version=$(get_version "externals.critools.version")
-crictl_tag_prefix="v"
+	crictl_repo=$(get_version "externals.critools.url")
+	crictl_version=$(get_version "externals.critools.version")
+	crictl_tag_prefix="v"
 
-crictl_url="${crictl_repo}/releases/download/v${crictl_version}/crictl-${crictl_tag_prefix}${crictl_version}-linux-$(${script_dir}/kata-arch.sh -g).tar.gz"
-curl -Ls "$crictl_url" | sudo tar xfz - -C /usr/local/bin
+	crictl_url="${crictl_repo}/releases/download/v${crictl_version}/crictl-${crictl_tag_prefix}${crictl_version}-linux-$(${script_dir}/kata-arch.sh -g).tar.gz"
+	curl -Ls "$crictl_url" | sudo tar xfz - -C /usr/local/bin
 }
 
 install_from_static_tarball || install_from_source
 
+if [ "${SNAPSHOTTER}" == "nydus" ]; then
+	install_vanilla_from_static_tarball
+fi
+
 install_cri-tools
 
 sudo systemctl daemon-reload
diff --git a/.ci/lib.sh b/.ci/lib.sh
index 9505aba8f..fa3699928 100755
--- a/.ci/lib.sh
+++ b/.ci/lib.sh
@@ -398,6 +398,9 @@ cleanup_nydus_snapshotter_dependencies() {
 	if [ -f "/usr/local/bin/nydus-image" ]; then
 		rm -f "/usr/local/bin/nydus-image"
 	fi
+	if [ -f "/usr/local/bin/containerd-vanilla" ]; then
+		rm -f "/usr/local/bin/containerd-vanilla"
+	fi
 }
 
 gen_clean_arch() {
diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh
index 9ec87e02c..6f6acc052 100644
--- a/integration/confidential/lib.sh
+++ b/integration/confidential/lib.sh
@@ -214,7 +214,7 @@ configure_cc_containerd() {
 	# Even if we are not saving the original file it is a good idea to
 	# restart containerd because it might be in an inconsistent state here.
 	sudo systemctl stop containerd
-	sleep 5
+	sleep 10
 	[ -n "$saved_containerd_conf_file" ] && \
 		sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
 	restart_containerd
@@ -476,18 +476,18 @@ is_containerd_support_per_runtime_snapshotter() {
 
 set_vanilla_containerd() {
 	sudo systemctl stop containerd
-	sleep 5
-	sudo mv /usr/local/bin/containerd /usr/local/bin/containerd-coco
-	sudo cp /usr/local/bin/containerd-vanilla /usr/local/bin/containerd
+	sleep 10
+	sudo mv -f /usr/local/bin/containerd /usr/local/bin/containerd-coco
+	sudo cp -f /usr/local/bin/containerd-vanilla /usr/local/bin/containerd
 	echo "vanilla containerd version: $(containerd --version | awk '{print $3}')"
 	restart_containerd
 }
 
 unset_vanilla_containerd() {
 	sudo systemctl stop containerd
-	sleep 5
+	sleep 10
 	sudo rm -f /usr/local/bin/containerd 
-	sudo mv /usr/local/bin/containerd-coco /usr/local/bin/containerd
+	sudo mv -f /usr/local/bin/containerd-coco /usr/local/bin/containerd
 	echo "coco containerd version: $(containerd --version | awk '{print $3}')"
 	restart_containerd
 }