diff --git a/.ci/ci_job_flags.sh b/.ci/ci_job_flags.sh index 8d5a52536..851215805 100755 --- a/.ci/ci_job_flags.sh +++ b/.ci/ci_job_flags.sh @@ -37,6 +37,9 @@ init_ci_flags() { # Build Kata for Confidential Containers # Values: "yes|no" export KATA_BUILD_CC="no" + # Use the forked version of containerd for Confidential Containers + # Valyes: "yes|no" + export FORKED_CONTAINERD="no" # Hypervisor to use export KATA_HYPERVISOR="" # Install k8s @@ -119,6 +122,7 @@ case "${CI_JOB}" in "CC_CRI_CONTAINERD"|"CC_CRI_CONTAINERD_K8S") # Export any CC specific environment variables export KATA_BUILD_CC="yes" + export FORKED_CONTAINERD="yes" export MEASURED_ROOTFS="yes" export AA_KBC="offline_fs_kbc" if [[ "${CI_JOB}" =~ K8S ]]; then @@ -133,6 +137,7 @@ case "${CI_JOB}" in export CRI_RUNTIME="containerd" export KATA_HYPERVISOR="qemu" export KATA_BUILD_CC="yes" + export FORKED_CONTAINERD="yes" export AA_KBC="offline_fs_kbc" export TEST_INITRD="yes" if [[ "${CI_JOB}" =~ K8S ]]; then @@ -155,6 +160,7 @@ case "${CI_JOB}" in export CRI_RUNTIME="containerd" export KATA_HYPERVISOR="qemu" export KATA_BUILD_CC="yes" + export FORKED_CONTAINERD="yes" export MEASURED_ROOTFS="yes" export AA_KBC="cc_kbc" export TEE_TYPE="tdx" @@ -174,6 +180,7 @@ case "${CI_JOB}" in export KUBERNETES="yes" export AA_KBC="offline_fs_kbc" export KATA_BUILD_CC="yes" + export FORKED_CONTAINERD="yes" export MEASURED_ROOTFS="yes" if [[ "${CI_JOB}" =~ TDX ]]; then export TEE_TYPE="tdx" @@ -199,6 +206,7 @@ case "${CI_JOB}" in export KATA_HYPERVISOR="cloud-hypervisor" # Export any CC specific environment variables export KATA_BUILD_CC="yes" + export FORKED_CONTAINERD="yes" export MEASURED_ROOTFS="yes" export AA_KBC="offline_fs_kbc" ;; diff --git a/.ci/install_cri_containerd.sh b/.ci/install_cri_containerd.sh index d13650876..ad5a11a80 100755 --- a/.ci/install_cri_containerd.sh +++ b/.ci/install_cri_containerd.sh @@ -25,7 +25,11 @@ source "${script_dir}/lib.sh" CONTAINERD_OS=$(go env GOOS) CONTAINERD_ARCH=$(go env GOARCH) -containerd_tarball_version=$(get_version "externals.containerd.version") +if [ "${FORKED_CONTAINERD}" = "yes" ]; then + containerd_tarball_version=$(get_version "externals.containerd.forked.version") +else + containerd_tarball_version=$(get_version "externals.containerd.upstream.version") +fi containerd_version=${containerd_tarball_version#v} @@ -38,7 +42,11 @@ fi install_from_source() { echo "Trying to install containerd from source" ( - containerd_repo=$(get_version "externals.containerd.url") + if [ "${FORKED_CONTAINERD}" = "yes" ]; then + containerd_repo=$(get_version "externals.containerd.forked.url") + else + containerd_repo=$(get_version "externals.containerd.upstream.url") + fi cd ${GOPATH}/src/ git clone "https://${containerd_repo}.git" "${GOPATH}/src/${containerd_repo}" @@ -55,7 +63,11 @@ install_from_source() { install_from_static_tarball() { echo "Trying to install containerd from static tarball" - local tarball_url=$(get_version "externals.containerd.tarball_url") + if [ "${FORKED_CONTAINERD}" = "yes" ]; then + local tarball_url=$(get_version "externals.containerd.forked.tarball_url") + else + local tarball_url=$(get_version "externals.containerd.upstream.tarball_url") + fi local tarball_name="cri-containerd-cni-${containerd_version}-${CONTAINERD_OS}-${CONTAINERD_ARCH}.tar.gz" local url="${tarball_url}/${containerd_tarball_version}/${tarball_name}" diff --git a/integration/containerd/cri/integration-tests.sh b/integration/containerd/cri/integration-tests.sh index 2adc833c6..53682d679 100755 --- a/integration/containerd/cri/integration-tests.sh +++ b/integration/containerd/cri/integration-tests.sh @@ -18,7 +18,11 @@ source "${SCRIPT_PATH}/../../../.ci/lib.sh" # runc is installed in /usr/local/sbin/ add that path export PATH="$PATH:/usr/local/sbin" -containerd_tarball_version=$(get_version "externals.containerd.version") +if [ "${FORKED_CONTAINERD}" = "yes" ]; then + containerd_tarball_version=$(get_version "externals.containerd.forked.version") +else + containerd_tarball_version=$(get_version "externals.containerd.upstream.version") +fi # Runtime to be used for testing RUNTIME=${RUNTIME:-containerd-shim-kata-v2} @@ -48,7 +52,11 @@ SNAP_CI=${SNAP_CI:-""} CI=${CI:-""} containerd_shim_path="$(command -v containerd-shim)" -readonly cri_containerd_repo=$(get_version "externals.containerd.url") +if [ "${FORKED_CONTAINERD}" = "yes" ]; then + readonly cri_containerd_repo=$(get_version "externals.containerd.forked.url") +else + readonly cri_containerd_repo=$(get_version "externals.containerd.upstream.url") +fi readonly cri_containerd_repo_git="https://${cri_containerd_repo}.git" #containerd config file @@ -464,7 +472,11 @@ main() { git reset HEAD # In CCv0 we are using a fork of containerd, so pull the matching branch of this - containerd_branch=$(get_version "externals.containerd.branch") + if [ "${FORKED_CONTAINERD}" = "yes" ]; then + containerd_branch=$(get_version "externals.containerd.forked.branch") + else + containerd_branch=$(get_version "externals.containerd.upstream.version") + fi git checkout "${containerd_branch}" # switch to the default pause image set by containerd:1.6.x @@ -522,4 +534,4 @@ main() { popd } -main \ No newline at end of file +main diff --git a/integration/nydus/nydus_tests.sh b/integration/nydus/nydus_tests.sh index 91ab19743..92523d1a7 100755 --- a/integration/nydus/nydus_tests.sh +++ b/integration/nydus/nydus_tests.sh @@ -30,8 +30,8 @@ containerd_config_backup="/tmp/containerd.config.toml" # test image for container IMAGE="${IMAGE:-ghcr.io/dragonflyoss/image-service/alpine:nydus-latest}" -if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "cloud-hypervisor" ] && [ "$KATA_HYPERVISOR" != "dragonball" ]; then - echo "Skip nydus test for $KATA_HYPERVISOR, it only works for QEMU/CLH/DB now." +if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "cloud-hypervisor" ]; then + echo "Skip nydus test for $KATA_HYPERVISOR, it only works for QEMU/CLH. now." exit 0 fi @@ -67,18 +67,12 @@ function setup_nydus() { # Config nydus snapshotter sudo -E cp "$dir_path/nydusd-config.json" /etc/ + sudo -E cp "$dir_path/snapshotter-config.toml" /etc/ # start nydus-snapshotter nohup /usr/local/bin/containerd-nydus-grpc \ - --config-path /etc/nydusd-config.json \ - --shared-daemon \ - --log-level debug \ - --root /var/lib/containerd/io.containerd.snapshotter.v1.nydus \ - --cache-dir /var/lib/nydus/cache \ - --nydusd-path /usr/local/bin/nydusd \ - --nydusimg-path /usr/local/bin/nydus-image \ - --disable-cache-manager true \ - --enable-nydus-overlayfs true \ + --config /etc/snapshotter-config.toml \ + --nydusd-config /etc/nydusd-config.json \ --log-to-stdout >/dev/null 2>&1 & } @@ -142,12 +136,22 @@ function config_containerd() { EOF } +function check_nydus_snapshotter_exist() { + bin="containerd-nydus-grpc" + if pgrep -f "$bin" >/dev/null; then + echo "nydus-snapshotter is running" + else + die "nydus-snapshotter is not running" + fi +} + function setup() { setup_nydus config_kata config_containerd restart_containerd_service check_processes + check_nydus_snapshotter_exist extract_kata_env } diff --git a/integration/nydus/nydusd-config.json b/integration/nydus/nydusd-config.json index 654b39353..4b75426d7 100644 --- a/integration/nydus/nydusd-config.json +++ b/integration/nydus/nydusd-config.json @@ -3,17 +3,13 @@ "backend": { "type": "registry", "config": { - "scheme": "https", "timeout": 5, "connect_timeout": 5, "retry_limit": 2 } }, "cache": { - "type": "blobcache", - "config": { - "work_dir": "/var/lib/nydus/cache" - } + "type": "blobcache" } }, "mode": "direct", @@ -22,6 +18,8 @@ "enable_xattr": true, "fs_prefetch": { "enable": true, - "threads_count": 2 + "threads_count": 8, + "merging_size": 1048576, + "prefetch_all": true } -} +} \ No newline at end of file diff --git a/integration/nydus/snapshotter-config.toml b/integration/nydus/snapshotter-config.toml new file mode 100644 index 000000000..7b668236c --- /dev/null +++ b/integration/nydus/snapshotter-config.toml @@ -0,0 +1,128 @@ +version = 1 +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" +daemon_mode = "dedicated" +# Whether snapshotter should try to clean up resources when it is closed +cleanup_on_close = false + +[system] +# Snapshotter's debug and trace HTTP server interface +enable = true +# Unix domain socket path where system controller is listening on +address = "/run/containerd-nydus/system.sock" + +[system.debug] +# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started. +# This option specifies the profile duration when nydusd is downloading and uncomproessing data. +daemon_cpu_profile_duration_secs = 5 +# Enable by assigning an address, empty indicates pprof server is disabled +pprof_address = "" + +[daemon] +# Specify a configuration file for nydusd +nydusd_config = "/etc/nydusd-config.json" +nydusd_path = "/usr/local/bin/nydusd" +nydusimage_path = "/usr/local/bin/nydus-image" +# fusedev or fscache +fs_driver = "fusedev" +# How to process when daemon dies: "none", "restart" or "failover" +recover_policy = "restart" +# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024]. +# Setting to 0 will use the default configuration of nydusd. +threads_number = 4 +# Log rotation size for nydusd, in unit MB(megabytes) +log_rotation_size = 100 + + +[cgroup] +# Whether to use separate cgroup for nydusd. +enable = true +# The memory limit for nydusd cgroup, which contains all nydusd processes. +# Percentage is supported as well, please ensure it is end with "%". +# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%". +memory_limit = "" + +[log] +# Print logs to stdout rather than logging files +log_to_stdout = false +# Snapshotter's log level +level = "info" +log_rotation_compress = true +log_rotation_local_time = true +# Max number of days to retain logs +log_rotation_max_age = 7 +log_rotation_max_backups = 5 +# In unit MB(megabytes) +log_rotation_max_size = 100 + +[metrics] +# Enable by assigning an address, empty indicates metrics server is disabled +address = ":9110" + +[remote] +convert_vpc_registry = false + +[remote.mirrors_config] +# Snapshotter will overwrite daemon's mirrors configuration +# if the values loaded from this driectory are not null before starting a daemon. +# Set to "" or an empty directory to disable it. +#dir = "/etc/nydus/certs.d" + +[remote.auth] +# Fetch the private registry auth by listening to K8s API server +enable_kubeconfig_keychain = false +# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`) +kubeconfig_path = "" +# Fetch the private registry auth as CRI image service proxy +enable_cri_keychain = false +# the target image service when using image proxy +#image_service_address = "/run/containerd/containerd.sock" + +[snapshot] +# Let containerd use nydus-overlayfs mount helper +enable_nydus_overlayfs = true +# Insert Kata Virtual Volume option to `Mount.Options` +enable_kata_volume = false +# Whether to remove resources when a snapshot is removed +sync_remove = false + +[cache_manager] +disable = false +gc_period = "24h" +# Directory to host cached files +cache_dir = "" + +[image] +public_key_file = "" +validate_signature = false + +# The configuraions for features that are not production ready +[experimental] +# Whether to enable stargz support +enable_stargz = false +# Whether to enable referrers support +# The option enables trying to fetch the Nydus image associated with the OCI image and run it. +# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers +enable_referrer_detect = false +[experimental.tarfs] +# Whether to enable nydus tarfs mode. Tarfs is supported by: +# - The EROFS filesystem driver since Linux 6.4 +# - Nydus Image Service release v2.3 +enable_tarfs = false +# Mount rafs on host by loopdev and EROFS +mount_tarfs_on_host = false +# Only enable nydus tarfs mode for images with `tarfs hint` label when true +tarfs_hint = false +# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default +max_concurrent_proc = 0 +# Mode to export tarfs images: +# - "none" or "": do not export tarfs +# - "layer_verity_only": only generate disk verity information for a layer blob +# - "image_verity_only": only generate disk verity information for all blobs of an image +# - "layer_block": generate a raw block disk image with tarfs for a layer +# - "image_block": generate a raw block disk image with tarfs for an image +# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info +# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info +export_mode = "" \ No newline at end of file