diff --git a/.ci/hypervisors/firecracker/configuration_firecracker.yaml b/.ci/hypervisors/firecracker/configuration_firecracker.yaml new file mode 100755 index 000000000..696dc623b --- /dev/null +++ b/.ci/hypervisors/firecracker/configuration_firecracker.yaml @@ -0,0 +1,44 @@ +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# We need to skip some docker integration tests as they are not +# running correctly using Firecracker. We will skip them using +# the ginkgo flag 'skip=REGEXP'. +test: + - docker +docker: + Describe: + - restart + - docker exec + - capabilities + - package manager update test + - build with docker + - inspect + - docker top + - users and groups + - terminal with docker + - docker commit + - ulimits + - docker cp with volume attached + - load with docker + - docker volume + - docker env + - CPUs and CPU set + - docker exit code + - run container with docker + - run hot plug block devices + - pause with docker + - Update number of CPUs + - docker cp + - docker privileges + - diff + - Hot plug CPUs + - Update CPU constraints + - memory constraints + - Hotplug memory when create containers + - run container and update its memory constraints + Context: + - remove bind-mount source before container exits + It: diff --git a/.ci/hypervisors/firecracker/filter_docker_firecracker.sh b/.ci/hypervisors/firecracker/filter_docker_firecracker.sh new file mode 100755 index 000000000..6fc4221b6 --- /dev/null +++ b/.ci/hypervisors/firecracker/filter_docker_firecracker.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -e + +cidir=$(dirname "$0") +source "${cidir}/../../lib.sh" + +test_config_file="${cidir}/configuration_firecracker.yaml" + +describe_skip_flag="docker.Describe" +context_skip_flag="docker.Context" +it_skip_flag="docker.It" + +# value for '-skip' in ginkgo +_skip_options=() + +filter_and_build() { + local dependency="$1" + local array_docker=$("${GOPATH}/bin/yq" read "${test_config_file}" "${dependency}") + [ "${array_docker}" = "null" ] && return + mapfile -t _array_docker <<< "${array_docker}" + for entry in "${_array_docker[@]}" + do + _skip_options+=("${entry#- }|") + done +} + +main() { + # Check if yq is installed + [ -z "$(command -v yq)" ] && install_yq + + # Build skip option based on Describe block + filter_and_build "${describe_skip_flag}" + + # Build skip option based on context block + filter_and_build "${context_skip_flag}" + + # Build skip option based on it block + filter_and_build "${it_skip_flag}" + + skip_options=$(IFS= ; echo "${_skip_options[*]}") + + echo "${skip_options%|}" +} + +main diff --git a/.ci/install_firecracker.sh b/.ci/install_firecracker.sh index 2ea9053e8..55714fbc0 100755 --- a/.ci/install_firecracker.sh +++ b/.ci/install_firecracker.sh @@ -11,6 +11,7 @@ set -o pipefail cidir=$(dirname "$0") arch=$("${cidir}"/kata-arch.sh -d) source "${cidir}/lib.sh" +KATA_DEV_MODE="${KATA_DEV_MODE:-false}" if [ "$arch" != "x86_64" ]; then die "Static binaries for Firecracker only available with x86_64." @@ -26,17 +27,23 @@ if [ "$docker_version" != "18.06" ]; then die "Firecracker hypervisor only works with docker 18.06" fi -# This is the initial release of Kata -# Containers that introduces support for -# the Firecracker hypervisor -release_version="1.5.0-rc2" -file_name="kata-fc-static-${release_version}-${arch}.tar.gz" -url="https://github.com/kata-containers/runtime/releases/download/${release_version}/${file_name}" -echo "Get static binaries from release version ${release_version}" -curl -OL ${url} +# Get url for firecracker from runtime/versions.yaml +firecracker_repo=$(get_version "assets.hypervisor.firecracker.url") +[ -n "$firecracker_repo" ] || die "failed to get firecracker repo" +firecracker_repo=${firecracker_repo/https:\/\//} -echo "Decompress binaries from release version ${release_version}" -sudo tar -xvf ${file_name} -C / +# Get version for firecracker from runtime/versions.yaml +firecracker_version=$(get_version "assets.hypervisor.firecracker.version") +[ -n "$firecracker_version" ] || die "failed to get firecracker version" + +# Get firecracker +go get -d ${firecracker_repo} || true +# Checkout to specific version +pushd "${GOPATH}/src/${firecracker_repo}" +git checkout tags/${firecracker_version} +./tools/devtool --unattended build --release -- --features vsock +sudo install ${GOPATH}/src/${firecracker_repo}/build/release/firecracker /usr/bin/ +popd echo "Install and configure docker" docker_configuration_path="/etc/docker" @@ -51,10 +58,7 @@ docker_configuration_file=$docker_configuration_path/daemon.json # is required driver="devicemapper" -# From decompressing the tarball, all the files are placed within -# /opt/kata. The runtime configuration is expected to land at -# /opt/kata/share/defaults/kata-containers/configuration.toml -path="/opt/kata/bin/kata-runtime" +path="/usr/local/bin/kata-runtime" if [ -f $docker_configuration_file ]; then # Check devicemapper flag @@ -67,7 +71,7 @@ else cat <<-EOF | sudo tee "$docker_configuration_file" { "runtimes": { - "kata": { + "kata-runtime": { "path": "${path}" } }, diff --git a/.ci/install_go.sh b/.ci/install_go.sh index a5bac2224..2c990b374 100755 --- a/.ci/install_go.sh +++ b/.ci/install_go.sh @@ -108,7 +108,10 @@ case "$(arch)" in "ppc64le") goarch=ppc64le ;; - + + "s390x") + goarch=s390x + ;; "*") die "Arch $(arch) not supported" ;; diff --git a/.ci/install_kata.sh b/.ci/install_kata.sh index 3238f8fc3..f464c11d8 100755 --- a/.ci/install_kata.sh +++ b/.ci/install_kata.sh @@ -12,7 +12,6 @@ set -o pipefail cidir=$(dirname "$0") source /etc/os-release || source /usr/lib/os-release source "${cidir}/lib.sh" - KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" echo "Install kata-containers image" @@ -21,7 +20,7 @@ echo "Install kata-containers image" echo "Install Kata Containers Kernel" "${cidir}/install_kata_kernel.sh" -if [ "$KATA_HYPERVISOR" == firecracker ]; then +if [ "$KATA_HYPERVISOR" == "firecracker" ]; then echo "Install Firecracker" "${cidir}/install_firecracker.sh" else diff --git a/.ci/install_qemu.sh b/.ci/install_qemu.sh index 41e2d82e4..b6a501c90 100755 --- a/.ci/install_qemu.sh +++ b/.ci/install_qemu.sh @@ -11,6 +11,7 @@ cidir=$(dirname "$0") source "${cidir}/lib.sh" source /etc/os-release || source /usr/lib/os-release +CURRENT_QEMU_BRANCH=$(get_version "assets.hypervisor.qemu-lite.branch") CURRENT_QEMU_COMMIT=$(get_version "assets.hypervisor.qemu-lite.commit") PACKAGED_QEMU="qemu-lite" QEMU_ARCH=$(${cidir}/kata-arch.sh -d) @@ -51,20 +52,21 @@ install_packaged_qemu() { } build_and_install_qemu() { - QEMU_REPO=$(get_version "assets.hypervisor.qemu-lite.url") + QEMU_REPO_URL=$(get_version "assets.hypervisor.qemu-lite.url") # Remove 'https://' from the repo url to be able to clone the repo using 'go get' - QEMU_REPO=${QEMU_REPO/https:\/\//} + QEMU_REPO=${QEMU_REPO_URL/https:\/\//} PACKAGING_REPO="github.com/kata-containers/packaging" QEMU_CONFIG_SCRIPT="${GOPATH}/src/${PACKAGING_REPO}/scripts/configure-hypervisor.sh" - go get -d "${QEMU_REPO}" || true + mkdir -p "${GOPATH}/src" + git clone --branch "$CURRENT_QEMU_BRANCH" --single-branch "${QEMU_REPO_URL}" "${GOPATH}/src/${QEMU_REPO}" go get -d "$PACKAGING_REPO" || true pushd "${GOPATH}/src/${QEMU_REPO}" git fetch git checkout "$CURRENT_QEMU_COMMIT" - [ -d "capstone" ] || git clone https://github.com/qemu/capstone.git capstone - [ -d "ui/keycodemapdb" ] || git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb + [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone.git capstone + [ -n "$(ls -A ui/keycodemapdb)" ] || git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb # Apply required patches QEMU_PATCHES_PATH="${GOPATH}/src/${PACKAGING_REPO}/obs-packaging/qemu-lite/patches" @@ -91,25 +93,31 @@ if [ -f "${cidir}/${QEMU_ARCH}/lib_install_qemu_${QEMU_ARCH}.sh" ]; then fi main() { - if [ "$QEMU_ARCH" == "x86_64" ]; then - packaged_qemu_commit=$(get_packaged_qemu_commit) - short_current_qemu_commit=${CURRENT_QEMU_COMMIT:0:10} - if [ "$packaged_qemu_commit" == "$short_current_qemu_commit" ]; then - # If installing packaged qemu from OBS fails, - # then build and install it from sources. - install_packaged_qemu || build_and_install_qemu - else - build_and_install_qemu - fi - elif [ "$QEMU_ARCH" == "aarch64" ] || [ "$QEMU_ARCH" == "ppc64le" ]; then - packaged_qemu_version=$(get_packaged_qemu_version) - short_current_qemu_version=${CURRENT_QEMU_VERSION#*-} - if [ "$packaged_qemu_version" == "$short_current_qemu_version" ] && [ -z "${CURRENT_QEMU_COMMIT}" ]; then - install_packaged_qemu || build_and_install_qemu - else - build_and_install_qemu - fi - fi + case "$QEMU_ARCH" in + "x86_64") + packaged_qemu_commit=$(get_packaged_qemu_commit) + short_current_qemu_commit=${CURRENT_QEMU_COMMIT:0:10} + if [ "$packaged_qemu_commit" == "$short_current_qemu_commit" ]; then + # If installing packaged qemu from OBS fails, + # then build and install it from sources. + install_packaged_qemu || build_and_install_qemu + else + build_and_install_qemu + fi + ;; + "aarch64"|"ppc64le"|"s390x") + packaged_qemu_version=$(get_packaged_qemu_version) + short_current_qemu_version=${CURRENT_QEMU_VERSION#*-} + if [ "$packaged_qemu_version" == "$short_current_qemu_version" ] && [ -z "${CURRENT_QEMU_COMMIT}" ] || [ "${QEMU_ARCH}" == "s390x" ]; then + install_packaged_qemu || build_and_install_qemu + else + build_and_install_qemu + fi + ;; + *) + die "Architecture $QEMU_ARCH not supported" + ;; + esac } main diff --git a/.ci/install_runtime.sh b/.ci/install_runtime.sh index 007375c37..110edce02 100755 --- a/.ci/install_runtime.sh +++ b/.ci/install_runtime.sh @@ -11,7 +11,6 @@ cidir=$(dirname "$0") source "${cidir}/lib.sh" source /etc/os-release || source /usr/lib/os-release - KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" # Modify the runtimes build-time defaults @@ -79,7 +78,7 @@ if [ "$USE_VSOCK" == "yes" ]; then fi fi -if [ "$KATA_HYPERVISOR" == qemu ]; then +if [ "$KATA_HYPERVISOR" == "qemu" ]; then echo "Add runtime as a new/default Docker runtime. Docker version \"$(docker --version)\" could change according to updates." docker_options="-D --add-runtime kata-runtime=/usr/local/bin/kata-runtime" echo "Add kata-runtime as a new/default Docker runtime." @@ -87,3 +86,9 @@ if [ "$KATA_HYPERVISOR" == qemu ]; then else echo "Kata runtime will not set as a default in Docker" fi + +if [ "$KATA_HYPERVISOR" == "firecracker" ]; then + echo "Enable firecracker configuration.toml" + path="/usr/share/defaults/kata-containers" + sudo mv ${path}/configuration-fc.toml ${path}/configuration.toml +fi diff --git a/.ci/lib.sh b/.ci/lib.sh index 05533dcca..60cb770b8 100755 --- a/.ci/lib.sh +++ b/.ci/lib.sh @@ -15,6 +15,9 @@ export KATA_RUNTIME=${KATA_RUNTIME:-kata-runtime} # more formats). export KATA_DOCKER_TIMEOUT=30 +# Ensure GOPATH set +export GOPATH=${GOPATH:-$(go env GOPATH)} + tests_repo="${tests_repo:-github.com/kata-containers/tests}" lib_script="${GOPATH}/src/${tests_repo}/lib/common.bash" source "${lib_script}" @@ -86,7 +89,6 @@ function build_and_install() { } function install_yq() { - GOPATH=${GOPATH:-${HOME}/go} local yq_path="${GOPATH}/bin/yq" local yq_pkg="github.com/mikefarah/yq" [ -x "${GOPATH}/bin/yq" ] && return @@ -153,7 +155,6 @@ function get_dep_from_yaml_db(){ function get_version(){ dependency="$1" - GOPATH=${GOPATH:-${HOME}/go} runtime_repo="github.com/kata-containers/runtime" runtime_repo_dir="$GOPATH/src/${runtime_repo}" versions_file="${runtime_repo_dir}/versions.yaml" @@ -177,13 +178,6 @@ function get_test_version(){ get_dep_from_yaml_db "${db}" "${dependency}" } -function check_gopath() { - # Verify GOPATH is set - if [ -z "$GOPATH" ]; then - export GOPATH=$(go env GOPATH) - fi -} - function waitForProcess(){ wait_time="$1" sleep_time="$2" @@ -226,7 +220,7 @@ delete_stale_docker_resource() local mount_point_union=$(mount | grep "${stale_docker_mount_point}" | awk '{print $3}') if [ -n "${mount_point_union}" ]; then while IFS='$\n' read mount_point; do - sudo umount "${mount_point}" + [ -n "$(grep "${mount_point}" "/proc/mounts")" ] && sudo umount -R "${mount_point}" done <<< "${mount_point_union}" fi done diff --git a/.ci/ppc64le/configuration_ppc64le.yaml b/.ci/ppc64le/configuration_ppc64le.yaml new file mode 100644 index 000000000..a5a5f202b --- /dev/null +++ b/.ci/ppc64le/configuration_ppc64le.yaml @@ -0,0 +1,24 @@ +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +# for now, not all integration test suites are fully passed in ppc64le. +# some need to be tested, and some need to be refined. +# sequence of 'test' holds supported integration tests components. +test: + - functional + - docker + - docker-compose + +# for now, not all test suites under docker integration are fully passed in aarch64. +# some need to be tested, and some need to be refined. +# ginkgo offers '-skip=REGEXP' flag to skip specific ones. +# you can use infos from docker.Describe, docker.Context or docker.It to point to +# specific test specs or whole container of specs. +docker: + Describe: + - Update CPU set + Context: + It: + - should have the right number of vCPUs diff --git a/.ci/ppc64le/filter_docker_ppc64le.sh b/.ci/ppc64le/filter_docker_ppc64le.sh new file mode 100755 index 000000000..8531dd516 --- /dev/null +++ b/.ci/ppc64le/filter_docker_ppc64le.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +set -e + +GOPATH_LOCAL="${GOPATH%%:*}" +kata_dir="${GOPATH_LOCAL}/src/github.com/kata-containers" +test_dir="${kata_dir}/tests" +ci_dir="${test_dir}/.ci" +test_config_file="${ci_dir}/ppc64le/configuration_ppc64le.yaml" + +describe_skip_flag="docker.Describe" +context_skip_flag="docker.Context" +it_skip_flag="docker.It" + +# value for '-skip' in ginkgo +_skip_options=() + +source "${ci_dir}/lib.sh" + +filter_and_build() +{ + local dependency="$1" + local array_docker=$("${GOPATH_LOCAL}/bin/yq" read "${test_config_file}" "${dependency}") + [ "${array_docker}" = "null" ] && return + mapfile -t _array_docker <<< "${array_docker}" + for entry in "${_array_docker[@]}" + do + _skip_options+=("${entry#- }|") + done +} + +main() +{ + # install yq if not exist + [ -z "$(command -v yq)" ] && install_yq + # build skip option based on Describe block + filter_and_build "${describe_skip_flag}" + + # build skip option based on context block + filter_and_build "${context_skip_flag}" + + # build skip option based on it block + filter_and_build "${it_skip_flag}" + + skip_options=$(IFS= ; echo "${_skip_options[*]}") + + echo "${skip_options%|}" +} + +main diff --git a/.ci/ppc64le/filter_test_ppc64le.sh b/.ci/ppc64le/filter_test_ppc64le.sh new file mode 100755 index 000000000..b068b3c8f --- /dev/null +++ b/.ci/ppc64le/filter_test_ppc64le.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +set -e + +GOPATH_LOCAL="${GOPATH%%:*}" +kata_dir="${GOPATH_LOCAL}/src/github.com/kata-containers" +test_dir="${kata_dir}/tests" +ci_dir="${test_dir}/.ci" +test_config_file="${ci_dir}/ppc64le/configuration_ppc64le.yaml" + +test_filter_flag="test" + +_test_union=() + +source "${ci_dir}/lib.sh" + +main() +{ + # install yq if not exist + [ -z "$(command -v yq)" ] && install_yq + local array_test=$("${GOPATH_LOCAL}/bin/yq" read "${test_config_file}" "${test_filter_flag}") + [ "${array_test}" = "null" ] && return + mapfile -t _array_test <<< "${array_test}" + for entry in "${_array_test[@]}" + do + _test_union+=("${entry#- }") + done + test_union=$(IFS=" "; echo "${_test_union[*]}") + echo "${test_union}" +} + +main diff --git a/.ci/ppc64le/lib_install_qemu_ppc64le.sh b/.ci/ppc64le/lib_install_qemu_ppc64le.sh index 4316e1e21..66c332dea 100755 --- a/.ci/ppc64le/lib_install_qemu_ppc64le.sh +++ b/.ci/ppc64le/lib_install_qemu_ppc64le.sh @@ -7,7 +7,8 @@ set -e CURRENT_QEMU_VERSION=$(get_version "assets.hypervisor.qemu.version") -PACKAGED_QEMU="qemu" +PACKAGED_QEMU="qemu-system-ppc" +BUILT_QEMU="qemu-system-ppc64" get_packaged_qemu_version() { if [ "$ID" == "ubuntu" ]; then @@ -66,7 +67,6 @@ build_and_install_qemu() { echo "Install Qemu" sudo -E make install - # Add link from /usr/local/bin to /usr/bin - sudo ln -sf $(command -v qemu-system-${QEMU_ARCH}) "/usr/bin/qemu-system-${QEMU_ARCH}" + sudo ln -sf $(command -v ${BUILT_QEMU}) "/usr/bin/qemu-system-${QEMU_ARCH}" popd } diff --git a/.ci/run.sh b/.ci/run.sh index 5b0c3419a..f98aa46ff 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -12,8 +12,6 @@ set -e cidir=$(dirname "$0") source "${cidir}/lib.sh" -check_gopath - export RUNTIME="kata-runtime" export CI_JOB="${CI_JOB:-default}" @@ -24,6 +22,16 @@ case "${CI_JOB}" in sudo -E PATH="$PATH" bash -c "make cri-containerd" sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make kubernetes" ;; + "FIRECRACKER") + echo "INFO: Running docker integration tests" + sudo -E PATH="$PATH" bash -c "make docker" + echo "INFO: Running soak test" + sudo -E PATH="$PATH" bash -c "make docker-stability" + echo "INFO: Running oci call test" + sudo -E PATH="$PATH" bash -c "make oci" + echo "INFO: Running networking tests" + sudo -E PATH="$PATH" bash -c "make network" + ;; *) echo "INFO: Running checks" sudo -E PATH="$PATH" bash -c "make check" diff --git a/.ci/s390x/clean_up_s390x.sh b/.ci/s390x/clean_up_s390x.sh new file mode 100755 index 000000000..98954c2bd --- /dev/null +++ b/.ci/s390x/clean_up_s390x.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +lib_script="${GOPATH}/src/${tests_repo}/.ci/lib.sh" +source "${lib_script}" + +gen_clean_arch || info "Arch cleanup scripts failed" diff --git a/.ci/s390x/configuration_s390x.yaml b/.ci/s390x/configuration_s390x.yaml new file mode 100644 index 000000000..4c04e675b --- /dev/null +++ b/.ci/s390x/configuration_s390x.yaml @@ -0,0 +1,22 @@ +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# +test: + - functional + - docker + +# Same model as is for aarch64 +docker: + Describe: + - CPUs and CPU set + - Update number of CPUs + - Hot plug CPUs + - Update CPU constraints + - Hotplug memory + - memory constraints + Context: + - remove bind-mount source before container exits + - run container exceeding memory constraints + It: diff --git a/.ci/s390x/filter_docker_s390x.sh b/.ci/s390x/filter_docker_s390x.sh new file mode 100755 index 000000000..d664def3b --- /dev/null +++ b/.ci/s390x/filter_docker_s390x.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +GOPATH_LOCAL="${GOPATH%%:*}" +kata_dir="${GOPATH_LOCAL}/src/github.com/kata-containers" +test_dir="${kata_dir}/tests" +ci_dir="${test_dir}/.ci" +test_config_file="${ci_dir}/s390x/configuration_s390x.yaml" + +describe_skip_flag="docker.Describe" +context_skip_flag="docker.Context" +it_skip_flag="docker.It" + +# value for '-skip' in ginkgo +_skip_options=() + +filter_and_build() +{ + local dependency="$1" + local array_docker=$("${GOPATH_LOCAL}/bin/yq" read "${test_config_file}" "${dependency}") + [ "${array_docker}" = "null" ] && return + mapfile -t _array_docker <<< "${array_docker}" + for entry in "${_array_docker[@]}" + do + _skip_options+=("${entry#- }|") + done +} + +main() +{ + # build skip option based on Describe block + filter_and_build "${describe_skip_flag}" + + # build skip option based on context block + filter_and_build "${context_skip_flag}" + + # build skip option based on it block + filter_and_build "${it_skip_flag}" + + skip_options=$(IFS= ; echo "${_skip_options[*]}") + + echo "${skip_options%|}" +} + +main diff --git a/.ci/s390x/filter_test_s390x.sh b/.ci/s390x/filter_test_s390x.sh new file mode 100755 index 000000000..4e5458ede --- /dev/null +++ b/.ci/s390x/filter_test_s390x.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +GOPATH_LOCAL="${GOPATH%%:*}" +kata_dir="${GOPATH_LOCAL}/src/github.com/kata-containers" +test_dir="${kata_dir}/tests" +ci_dir="${test_dir}/.ci" +test_config_file="${ci_dir}/s390x/configuration_s390x.yaml" + +test_filter_flag="test" + +_test_union=() + +main() +{ + local array_test=$("${GOPATH_LOCAL}/bin/yq" read "${test_config_file}" "${test_filter_flag}") + [ "${array_test}" = "null" ] && return + mapfile -t _array_test <<< "${array_test}" + for entry in "${_array_test[@]}" + do + _test_union+=("${entry#- }") + done + test_union=$(IFS=" "; echo "${_test_union[*]}") + echo "${test_union}" +} + +main diff --git a/.ci/s390x/lib_install_qemu_s390x.sh b/.ci/s390x/lib_install_qemu_s390x.sh new file mode 100755 index 000000000..c29833f95 --- /dev/null +++ b/.ci/s390x/lib_install_qemu_s390x.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +CURRENT_QEMU_VERSION=$(get_version "assets.hypervisor.qemu.version") +PACKAGED_QEMU="qemu" + +[ "$ID" == "ubuntu" ] || die "Unsupported distro: $ID" + +get_packaged_qemu_version() { + if [ "$ID" == "ubuntu" ]; then + sudo apt-get update > /dev/null + qemu_version=$(apt-cache madison $PACKAGED_QEMU \ + | awk '{print $3}' | cut -d':' -f2 | cut -d'+' -f1 | head -n 1 ) + fi + + if [ -z "$qemu_version" ]; then + die "unknown qemu version" + else + echo "${qemu_version}" + fi +} + +install_packaged_qemu() { + sudo apt install -y "$PACKAGED_QEMU" +} + +build_and_install_qemu() { + QEMU_REPO=$(get_version "assets.hypervisor.qemu.url") + # Remove 'https://' from the repo url to be able to clone the repo using 'go get' + QEMU_REPO_PATH=${QEMU_REPO/https:\/\//} + + PACKAGING_REPO="github.com/kata-containers/packaging" + QEMU_CONFIG_SCRIPT="${GOPATH}/src/${PACKAGING_REPO}/scripts/configure-hypervisor.sh" + + if [ ! -d "${GOPATH}/src/${QEMU_REPO_PATH}" ]; then + mkdir -p "${GOPATH}/src/${QEMU_REPO_PATH}" + pushd "${GOPATH}/src/${QEMU_REPO_PATH}" + chronic git clone "${QEMU_REPO}" "." + popd + fi + + go get -d "$PACKAGING_REPO" || true + + pushd "${GOPATH}/src/${QEMU_REPO_PATH}" + git fetch + git checkout "$CURRENT_QEMU_VERSION" + [ -d "capstone" ] || git clone https://github.com/qemu/capstone.git capstone + [ -d "ui/keycodemapdb" ] || git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb + + echo "Build Qemu" + "${QEMU_CONFIG_SCRIPT}" "qemu" | xargs ./configure + make -j $(nproc) + + echo "Install Qemu" + sudo -E make install + popd +} diff --git a/.ci/s390x/lib_kata_image_s390x.sh b/.ci/s390x/lib_kata_image_s390x.sh new file mode 100755 index 000000000..b536deea3 --- /dev/null +++ b/.ci/s390x/lib_kata_image_s390x.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +OSBUILDER_DISTRO="ubuntu" +AGENT_INIT="yes" + +#packaged kata agent haven't been supported in any mainstream distribution +get_packaged_agent_version() { + version="" + echo "$version" +} + +#packaged kata image haven't been supported in any mainstream distribution +install_packaged_image() { + info "installing packaged kata-image not supported in s390x" + return 1 +} diff --git a/.ci/s390x/lib_setup_s390x.sh b/.ci/s390x/lib_setup_s390x.sh new file mode 100755 index 000000000..a75323f8b --- /dev/null +++ b/.ci/s390x/lib_setup_s390x.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +filter_test_script="${cidir}/${arch}/filter_test_s390x.sh" + +check_test_union() +{ + local test_union=$(bash -f ${filter_test_script}) + flag="$1" + # regex match + [[ ${test_union} =~ ${flag} ]] && echo "true" + + echo "false" +} + +CRIO=$(check_test_union crio) +KUBERNETES=$(check_test_union kubernetes) +OPENSHIFT=$(check_test_union openshift) diff --git a/.ci/setup.sh b/.ci/setup.sh index 154906f2c..aca8c61d1 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -57,12 +57,14 @@ install_docker() { enable_nested_virtualization() { case "$arch" in - x86_64) - if [ "$CI" == true ] && grep -q "N" /sys/module/kvm_intel/parameters/nested 2>/dev/null; then + x86_64 | s390x) + kvm_arch="kvm" + [ ${arch} == "x86_64" ] && kvm_arch="kvm_intel" + if [ "$CI" == true ] && grep -q "N" /sys/module/$kvm_arch/parameters/nested 2>/dev/null; then echo "enable Nested Virtualization" - sudo modprobe -r kvm_intel - sudo modprobe kvm_intel nested=1 - if grep -q "N" /sys/module/kvm_intel/parameters/nested 2>/dev/null; then + sudo modprobe -r $kvm_arch + sudo modprobe $kvm_arch nested=1 + if grep -q "N" /sys/module/$kvm_arch/parameters/nested 2>/dev/null; then die "Failed to find or enable Nested virtualization" fi fi @@ -116,7 +118,6 @@ install_extra_tools() { } main() { - check_gopath setup_distro_env install_docker enable_nested_virtualization diff --git a/.ci/setup_env_debian.sh b/.ci/setup_env_debian.sh index 3cb44d655..d2915b3c0 100755 --- a/.ci/setup_env_debian.sh +++ b/.ci/setup_env_debian.sh @@ -22,7 +22,7 @@ echo "Install git" sudo -E apt install -y git echo "Install kata containers dependencies" -chronic sudo -E apt install -y libtool automake autotools-dev autoconf bc alien libpixman-1-dev coreutils +chronic sudo -E apt install -y libtool automake autotools-dev autoconf bc alien libpixman-1-dev coreutils parted echo "Install qemu dependencies" chronic sudo -E apt install -y libcap-dev libattr1-dev libcap-ng-dev librbd-dev diff --git a/.ci/static-checks.sh b/.ci/static-checks.sh index a280a30ce..2dea07fb7 100755 --- a/.ci/static-checks.sh +++ b/.ci/static-checks.sh @@ -473,7 +473,7 @@ check_docs() if [ "$specific_branch" != "true" ] then # If the URL is new on this PR, it cannot be checked. - echo "$new_urls" | grep -q "\<${url}\>" && \ + echo "$new_urls" | egrep -q "\<${url}\>" && \ info "ignoring new (but correct) URL: $url" && continue fi diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..e48c094af --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,13 @@ +# Copyright 2019 Intel Corporation. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Define any code owners for this repository. +# The code owners lists are used to help automatically enforce +# reviews and acks of the right groups on the right PRs. + +# Order in this file is important. Only the last match will be +# used. See https://help.github.com/articles/about-code-owners/ + +*.md @kata-containers/documentation + diff --git a/Makefile b/Makefile index 4cc7d7a3f..06265ef9e 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,10 @@ TIMEOUT := 60 UNION := functional docker crio docker-compose network netmon docker-stability oci openshift kubernetes swarm vm-factory entropy ramdisk shimv2 # skipped test suites for docker integration tests -SKIP := +FILTER_FILE = .ci/hypervisors/$(KATA_HYPERVISOR)/filter_docker_$(KATA_HYPERVISOR).sh +ifneq ($(wildcard $(FILTER_FILE)),) + SKIP := $(shell bash -f $(FILTER_FILE)) +endif # get arch ARCH := $(shell bash -c '.ci/kata-arch.sh -d') @@ -64,6 +67,7 @@ docker-stability: systemctl is-active --quiet docker || sudo systemctl start docker cd integration/stability && \ export ITERATIONS=2 && export MAX_CONTAINERS=20 && ./soak_parallel_rm.sh + cd integration/stability && ./bind_mount_linux.sh kubernetes: bash -f .ci/install_bats.sh diff --git a/arch/ppc64le-options.mk b/arch/ppc64le-options.mk new file mode 100644 index 000000000..805cc7284 --- /dev/null +++ b/arch/ppc64le-options.mk @@ -0,0 +1,10 @@ +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +# union for 'make test' +UNION := $(shell bash -f .ci/ppc64le/filter_test_ppc64le.sh) + +# skiped test suites for docker integration tests +SKIP := $(shell bash -f .ci/ppc64le/filter_docker_ppc64le.sh) diff --git a/arch/s390x-options.mk b/arch/s390x-options.mk new file mode 100755 index 000000000..04640c31e --- /dev/null +++ b/arch/s390x-options.mk @@ -0,0 +1,10 @@ +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +# union for 'make test' +UNION := $(shell bash -f .ci/s390x/filter_test_s390x.sh) + +# skiped test suites for docker integration tests +SKIP := $(shell bash -f .ci/s390x/filter_docker_s390x.sh) diff --git a/cmd/kata-manager/kata-manager.sh b/cmd/kata-manager/kata-manager.sh index 5b415f3ef..b9515ffb0 100755 --- a/cmd/kata-manager/kata-manager.sh +++ b/cmd/kata-manager/kata-manager.sh @@ -270,7 +270,7 @@ get_git_repo() fi info "getting repo $1 using git" - git clone "$repo_url" "$local_dest" + git clone "$repo_url" "$local_dest" || (rm -fr "$local_dest" && exit 1) } exec_document() @@ -349,25 +349,24 @@ cmd_remove_packages() info "removing packages" case "$distro" in - centos|fedora) + centos|fedora|opensuse|rhel|sles) packages=$(rpm -qa|egrep "${packages_regex}" || true) ;; - ubuntu) + debian|ubuntu) packages=$(dpkg-query -W -f='${Package}\n'|egrep "${packages_regex}" || true) ;; - *) - die "invalid distro: '$distro'" - ;; + *) die "invalid distro: '$distro'" ;; esac [ -z "$packages" ] && die "packages not installed" case "$distro" in - centos) sudo yum -y remove $packages ;; + centos|rhel) sudo yum -y remove $packages ;; + debian|ubuntu) sudo apt-get -y remove $packages ;; fedora) sudo dnf -y remove $packages ;; - ubuntu) sudo apt-get -y remove $packages ;; + opensuse|sles) sudo zypper remove -y $packages ;; esac } diff --git a/config.go b/config.go index d960707d3..26585eed8 100644 --- a/config.go +++ b/config.go @@ -70,6 +70,9 @@ const ( // DefaultHypervisor default hypervisor DefaultHypervisor = "qemu" + // FirecrackerHypervisor is firecracker + FirecrackerHypervisor = "firecracker" + // DefaultProxy default proxy DefaultProxy = "kata" diff --git a/data/config.json b/data/config.json index 167d42ee5..c8e2ebd0c 100644 --- a/data/config.json +++ b/data/config.json @@ -122,6 +122,7 @@ } ] }, + "cgroupsPath": "kata", "namespaces": [ { "type": "pid" diff --git a/integration/docker/cp_test.go b/integration/docker/cp_test.go index 818edb0fa..019b21047 100644 --- a/integration/docker/cp_test.go +++ b/integration/docker/cp_test.go @@ -123,7 +123,6 @@ var _ = Describe("docker cp with volume", func() { Context("check mount points", func() { It("should be removed", func() { - Skip("Issue: https://github.com/kata-containers/runtime/issues/794") file, err := ioutil.TempFile(os.TempDir(), "file") Expect(err).ToNot(HaveOccurred()) err = file.Close() diff --git a/integration/docker/cpu_test.go b/integration/docker/cpu_test.go index fe7ca8d6c..b4bb9823f 100644 --- a/integration/docker/cpu_test.go +++ b/integration/docker/cpu_test.go @@ -7,6 +7,7 @@ package docker import ( "fmt" "math" + "runtime" "strings" . "github.com/kata-containers/tests" @@ -312,7 +313,13 @@ func withCPUConstraintCheckPeriodAndQuota(cpus float64, fail bool) TableEntry { return Entry(fmt.Sprintf("quota/period should be equal to %.1f", cpus), cpus, fail) } -func withCPUSetConstraint(cpuset string, fail bool) TableEntry { +func withCPUSetConstraint(cpuset string, minCpusNeeded int, fail bool) TableEntry { + // test should fail when the actual number of cpus is less than the minimum number + // of cpus needed to run the test, for example cpuset=0-2 requires 3 cpus(0,1,2) + if runtime.NumCPU() < minCpusNeeded { + fail = true + } + return Entry(fmt.Sprintf("cpuset should be equal to %s", cpuset), cpuset, fail) } @@ -371,7 +378,9 @@ var _ = Describe("Update CPU constraints", func() { DescribeTable("Update CPU set", func(cpuset string, fail bool) { - runArgs = []string{"--rm", "--cpus=4", "--name", id, "-dt", DebianImage, "bash"} + // Use the actual number of CPUs + runArgs = []string{"--rm", fmt.Sprintf("--cpus=%d", runtime.NumCPU()), + "--name", id, "-dt", DebianImage, "bash"} _, _, exitCode = dockerRun(runArgs...) Expect(exitCode).To(BeZero()) @@ -388,15 +397,15 @@ var _ = Describe("Update CPU constraints", func() { Expect(exitCode).To(BeZero()) Expect(cpuset).To(Equal(strings.Trim(stdout, "\n\t "))) }, - withCPUSetConstraint("0", shouldNotFail), - withCPUSetConstraint("2", shouldNotFail), - withCPUSetConstraint("0-1", shouldNotFail), - withCPUSetConstraint("0-2", shouldNotFail), - withCPUSetConstraint("0-3", shouldNotFail), - withCPUSetConstraint("0,2", shouldNotFail), - withCPUSetConstraint("0,3", shouldNotFail), - withCPUSetConstraint("0,-2,3", shouldFail), - withCPUSetConstraint("-1-3", shouldFail), + withCPUSetConstraint("0", 1, shouldNotFail), + withCPUSetConstraint("2", 3, shouldNotFail), + withCPUSetConstraint("0-1", 2, shouldNotFail), + withCPUSetConstraint("0-2", 3, shouldNotFail), + withCPUSetConstraint("0-3", 4, shouldNotFail), + withCPUSetConstraint("0,2", 3, shouldNotFail), + withCPUSetConstraint("0,3", 4, shouldNotFail), + withCPUSetConstraint("0,-2,3", 0, shouldFail), + withCPUSetConstraint("-1-3", 0, shouldFail), ) }) diff --git a/integration/docker/docker.go b/integration/docker/docker.go index 4cee5a933..19e1292ea 100644 --- a/integration/docker/docker.go +++ b/integration/docker/docker.go @@ -36,9 +36,6 @@ const ( // FedoraImage is the fedora image FedoraImage = "fedora" - // CentosImage is the centos image - CentosImage = "centos" - // StressImage is the vish/stress image StressImage = "vish/stress" diff --git a/integration/docker/docker_arch_base.go b/integration/docker/docker_arch_base.go new file mode 100644 index 000000000..1efcc042a --- /dev/null +++ b/integration/docker/docker_arch_base.go @@ -0,0 +1,12 @@ +// +build !s390x + +// Copyright (c) 2019 IBM +// +// SPDX-License-Identifier: Apache-2.0 + +package docker + +const ( + // CentosImage is the centos image + CentosImage = "centos" +) diff --git a/integration/docker/docker_s390x.go b/integration/docker/docker_s390x.go new file mode 100644 index 000000000..4f0344082 --- /dev/null +++ b/integration/docker/docker_s390x.go @@ -0,0 +1,10 @@ +// Copyright (c) 2019 IBM +// +// SPDX-License-Identifier: Apache-2.0 + +package docker + +const ( + // On s390x there is not centos image, instead the clefos image could be used + CentosImage = "clefos" +) diff --git a/integration/docker/main_test.go b/integration/docker/main_test.go index 408dd0cfe..6658aff83 100644 --- a/integration/docker/main_test.go +++ b/integration/docker/main_test.go @@ -38,7 +38,7 @@ func TestIntegration(t *testing.T) { for _, i := range images { // vish/stress is single-arch image only for amd64 - if i == StressImage && runtime.GOARCH == "arm64" { + if i == StressImage && runtime.GOARCH != "amd64" { //check if vish/stress has already been built argsImage := []string{"--format", "'{{.Repository}}:{{.Tag}}'", StressImage} imagesStdout, _, imagesExitcode := dockerImages(argsImage...) diff --git a/integration/docker/run_test.go b/integration/docker/run_test.go index 7e71f2179..01a9f75bb 100644 --- a/integration/docker/run_test.go +++ b/integration/docker/run_test.go @@ -253,7 +253,6 @@ var _ = Describe("run host networking", func() { Context("Run with host networking", func() { It("should error out", func() { - Skip("Issue: https://github.com/kata-containers/runtime/issues/652") args = []string{"--name", id, "-d", "--net=host", DebianImage, "sh"} _, stderr, exitCode = dockerRun(args...) Expect(exitCode).NotTo(Equal(0)) diff --git a/integration/kubernetes/e2e_conformance/run.sh b/integration/kubernetes/e2e_conformance/run.sh new file mode 100755 index 000000000..abb8d8910 --- /dev/null +++ b/integration/kubernetes/e2e_conformance/run.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# This script runs the Sonobuoy e2e Conformance tests. +# Run this script once your K8s cluster is running. +# WARNING: it is prefered to use containerd as the +# runtime interface instead of cri-o as we have seen +# errors with cri-o that still need to be debugged. + +set -o errexit +set -o nounset +set -o pipefail + +export KUBECONFIG=$HOME/.kube/config +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../../../lib/common.bash" + +RUNTIME="${RUNTIME:-kata-runtime}" + +# Check if Sonobuoy is still running every 5 minutes. +WAIT_TIME=300 + +# Add a global timeout of 2 hours to stop the execution +# in case Sonobuoy gets hanged. +GLOBAL_TIMEOUT=$((WAIT_TIME*24)) + +create_kata_webhook() { + pushd "${SCRIPT_PATH}/../../../kata-webhook" >> /dev/null + # Create certificates for the kata webhook + ./create-certs.sh + + # Apply kata-webhook deployment + kubectl apply -f deploy/ + popd +} + +run_sonobuoy() { + sonobuoy_repo="github.com/heptio/sonobuoy" + go get -u "$sonobuoy_repo" + + # Run Sonobuoy e2e tests + info "Starting sonobuoy execution." + info "When using kata as k8s runtime, the tests take around 2 hours to finish." + sonobuoy run + + start_time=$(date +%s) + estimated_end_time=$((start_time + GLOBAL_TIMEOUT)) + + # Wait for the sonobuoy pod to be running. + kubectl wait --for condition=Ready pod sonobuoy -n heptio-sonobuoy + + while sonobuoy status | grep -Eq "running|pending" && [ "$(date +%s)" -le "$estimated_end_time" ]; do + info "sonobuoy still running, sleeping $WAIT_TIME seconds" + sleep "$WAIT_TIME" + done + + # Retrieve results + e2e_result_dir="$(mktemp -d /tmp/kata_e2e_results.XXXXX)" + sonobuoy retrieve "$e2e_result_dir" || \ + die "Couldn't retrieve sonobuoy results, please check status using: sonobuoy status" + pushd "$e2e_result_dir" >> /dev/null + + # Uncompress results + ls | grep tar.gz | xargs tar -xvf + e2e_result_log="${e2e_result_dir}/plugins/e2e/results/e2e.log" + info "Results of the e2e tests can be found on: $e2e_result_log" + + # If on CI, display the e2e log on the console. + [ "$CI" == true ] && cat "$e2e_result_log" + + # Check for Success message on the logs. + grep -aq " 0 Failed" "$e2e_result_log" + grep -aq "SUCCESS" "$e2e_result_log" && \ + info " k8s e2e conformance using Kata runtime finished successfully" + popd +} + +cleanup() { + # Remove sonobuoy execution pods + sonobuoy delete + info "Results directory $e2e_result_dir will not be deleted" +} + +main() { + if [ "$RUNTIME" == "kata-runtime" ]; then + create_kata_webhook + fi + run_sonobuoy + cleanup +} + +main diff --git a/integration/kubernetes/init.sh b/integration/kubernetes/init.sh index 0814bdd23..3df60803e 100755 --- a/integration/kubernetes/init.sh +++ b/integration/kubernetes/init.sh @@ -50,6 +50,7 @@ kubeadm_config_template="${SCRIPT_PATH}/kubeadm/config.yaml" kubeadm_config_file="$(mktemp --tmpdir kubeadm_config.XXXXXX.yaml)" sed -e "s|CRI_RUNTIME_SOCKET|${cri_runtime_socket}|" "${kubeadm_config_template}" > "${kubeadm_config_file}" +sed -i "s|KUBERNETES_VERSION|v${kubernetes_version/-*}|" "${kubeadm_config_file}" if [ "${use_runtime_class}" == true ]; then echo "Add RuntimeClass feature for apiserver in kubeadm config file" diff --git a/integration/kubernetes/k8s-memory.bats b/integration/kubernetes/k8s-memory.bats index 9551be144..d5cd847de 100644 --- a/integration/kubernetes/k8s-memory.bats +++ b/integration/kubernetes/k8s-memory.bats @@ -8,9 +8,10 @@ load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh" TEST_INITRD="${TEST_INITRD:-no}" issue="https://github.com/kata-containers/runtime/issues/1127" +memory_issue="https://github.com/kata-containers/runtime/issues/1249" setup() { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" export KUBECONFIG=/etc/kubernetes/admin.conf pod_name="memory-test" @@ -23,7 +24,7 @@ setup() { } @test "Exceeding memory constraints" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" memory_limit_size="50Mi" allocated_size="250M" @@ -41,7 +42,7 @@ setup() { } @test "Running within memory constraints" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" memory_limit_size="200Mi" allocated_size="100M" diff --git a/integration/kubernetes/k8s-qos-pods.bats b/integration/kubernetes/k8s-qos-pods.bats index 4bfba7c03..092529b25 100644 --- a/integration/kubernetes/k8s-qos-pods.bats +++ b/integration/kubernetes/k8s-qos-pods.bats @@ -8,9 +8,10 @@ load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh" TEST_INITRD="${TEST_INITRD:-no}" issue="https://github.com/kata-containers/runtime/issues/1127" +memory_issue="https://github.com/kata-containers/runtime/issues/1249" setup() { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" export KUBECONFIG=/etc/kubernetes/admin.conf if sudo -E kubectl get runtimeclass | grep kata; then @@ -21,7 +22,7 @@ setup() { } @test "Guaranteed QoS" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" pod_name="qos-test" @@ -36,7 +37,7 @@ setup() { } @test "Burstable QoS" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" pod_name="burstable-test" @@ -51,7 +52,7 @@ setup() { } @test "BestEffort QoS" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" pod_name="besteffort-test" # Create pod @@ -65,6 +66,6 @@ setup() { } teardown() { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" - sudo -E kubectl delete pod "$pod_name" + skip "test not working see: ${issue}, ${memory_issue}" + kubectl delete pod "$pod_name" } diff --git a/integration/kubernetes/kubeadm/config.yaml b/integration/kubernetes/kubeadm/config.yaml index b90559dfd..0a7e0c5aa 100644 --- a/integration/kubernetes/kubeadm/config.yaml +++ b/integration/kubernetes/kubeadm/config.yaml @@ -5,7 +5,7 @@ nodeRegistration: --- apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration -kubernetesVersion: v1.12.0 +kubernetesVersion: KUBERNETES_VERSION networking: dnsDomain: cluster.local podSubnet: 10.244.0.0/16 diff --git a/integration/network/disable_net/net_none.bats b/integration/network/disable_net/net_none.bats index 273f1feaa..0bfb7a5d2 100644 --- a/integration/network/disable_net/net_none.bats +++ b/integration/network/disable_net/net_none.bats @@ -10,8 +10,11 @@ load "${BATS_TEST_DIRNAME}/../../../lib/common.bash" IMAGE="busybox" PAYLOAD="tail -f /dev/null" NAME="test" +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" +issue="https://github.com/kata-containers/runtime/issues/1197" setup () { + [ "${KATA_HYPERVISOR}" = "firecracker" ] && skip "test not working see: ${issue}" clean_env # Check that processes are not running @@ -21,6 +24,7 @@ setup () { } @test "Disable_new_netns equal to false" { + [ "${KATA_HYPERVISOR}" = "firecracker" ] && skip "test not working see: ${issue}" extract_kata_env sudo sed -i 's/#disable_new_netns = true/disable_new_netns = false/g' ${RUNTIME_CONFIG_PATH} @@ -47,6 +51,7 @@ setup () { } @test "Disable net" { + [ "${KATA_HYPERVISOR}" = "firecracker" ] && skip "test not working see: ${issue}" extract_kata_env # Get the name of the network name at the configuration.toml @@ -80,6 +85,7 @@ setup () { } teardown() { + [ "${KATA_HYPERVISOR}" = "firecracker" ] && skip "test not working see: ${issue}" clean_env # Check that processes are not running diff --git a/integration/stability/bind_mount_linux.sh b/integration/stability/bind_mount_linux.sh new file mode 100755 index 000000000..006018875 --- /dev/null +++ b/integration/stability/bind_mount_linux.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# +# Copyright (c) 2019 Ning Lu +# +# SPDX-License-Identifier: Apache-2.0 +# +# This test will start a container with a bind mount +# and set bind propagation, the purpose of this +# test is to check if the container stops properly + +set -e + +cidir=$(dirname "$0") +testname="${0##*/}" +sysname=$(uname) + +if [ "${sysname}" != "Linux" ]; then + echo "Skip ${testname} on ${sysname}" + exit 0 +fi + +source "${cidir}/../../lib/common.bash" + +# Environment variables +IMAGE="${IMAGE:-busybox}" +CONTAINER_NAME="${CONTAINER_NAME:-test}" +PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}" +TMP_DIR=$(mktemp -d --tmpdir=/tmp ${testname}.XXX) +MOUNT_DIR="${TMP_DIR}/mount" +BIND_DST="${MOUNT_DIR}/dst" +BIND_SRC="${TMP_DIR}/src" +DOCKER_ARGS="-v ${MOUNT_DIR}:${MOUNT_DIR}:rslave" +CONTAINER_ID= + +# Set the runtime if not set already +RUNTIME="${RUNTIME:-kata-runtime}" + +function setup { + clean_env + docker run --runtime=${RUNTIME} -d ${DOCKER_ARGS} --name ${CONTAINER_NAME} ${IMAGE} ${PAYLOAD_ARGS} + CONTAINER_ID=$(docker ps -q -f "name=${CONTAINER_NAME}") +} + +function cmd_bind_mount { + mkdir -p ${BIND_SRC} + mkdir -p ${BIND_DST} + mount --bind ${BIND_SRC} ${BIND_DST} + docker rm -f ${CONTAINER_NAME} + + KATA_PROC=$(ps aux | grep ${CONTAINER_ID} | grep -v grep | tee) +} + +function clean_kata_proc { + kata_pids=$(echo -n "${KATA_PROC}" | awk '{print $2}') + [ -n "${kata_pids}" ] && echo "${kata_pids}" | xargs kill + + kata_mount=$(mount | grep ${CONTAINER_ID} | awk '{print $3}'| sort -r) + [ -n "${kata_mount}" ] && echo "${kata_mount}" | xargs -n1 umount + + rm -rf ${TMP_DIR} +} + +function check { + if [ -n "${KATA_PROC}" ]; then + clean_kata_proc + die "Left kata processes, quitting: ${KATA_PROC}" + fi +} + +function teardown { + clean_env + if mountpoint -q ${BIND_DST}; then + umount ${BIND_DST} + fi + rm -rf ${TMP_DIR} +} + +echo "Starting stability test: ${testname}" +setup + +echo "Running stability test: ${testname}" +cmd_bind_mount +check + +echo "Ending stability test: ${testname}" +teardown diff --git a/integration/stability/soak_parallel_rm.sh b/integration/stability/soak_parallel_rm.sh index 244c5fa3c..12a8ae3cb 100755 --- a/integration/stability/soak_parallel_rm.sh +++ b/integration/stability/soak_parallel_rm.sh @@ -41,12 +41,7 @@ MAX_CONTAINERS="${MAX_CONTAINERS:-110}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" -if [ "$KATA_HYPERVISOR" == "firecracker" ]; then - echo "Skip soak test on $KATA_HYPERVISOR (see: https://github.com/kata-containers/tests/issues/1029)" - exit -fi - -if [ "$ID" == debian ]; then +if [ "$ID" == "debian" ]; then echo "Skip soak test on ${ID} (see: https://github.com/kata-containers/runtime/issues/1132)" exit fi @@ -100,19 +95,21 @@ check_all_running() { # Only check for Kata components if we are using a Kata runtime if (( $check_kata_components )); then - # Check we have one proxy per container - how_many_proxys=$(pgrep -a -f ${PROXY_PATH} | wc -l) - if check_vsock_active; then - if (( ${how_many_proxys} != 0 )); then - echo "Wrong number of proxys running (${how_many_running} containers, ${how_many_proxys} proxys)" - echo "When using vsocks, the number of proxies should be Zero - stopping" - ((goterror++)) - fi - - else - if (( ${how_many_running} != ${how_many_proxys} )); then - echo "Wrong number of proxys running (${how_many_running} containers, ${how_many_proxys} proxys) - stopping" - ((goterror++)) + if [ "$KATA_HYPERVISOR" == "qemu" ]; then + # Check we have one proxy per container + how_many_proxys=$(pgrep -a -f ${PROXY_PATH} | wc -l) + if check_vsock_active; then + if (( ${how_many_proxys} != 0 )); then + echo "Wrong number of proxys running (${how_many_running} containers, ${how_many_proxys} proxys)" + echo "When using vsocks, the number of proxies should be Zero - stopping" + ((goterror++)) + fi + + else + if (( ${how_many_running} != ${how_many_proxys} )); then + echo "Wrong number of proxys running (${how_many_running} containers, ${how_many_proxys} proxys) - stopping" + ((goterror++)) + fi fi fi @@ -131,11 +128,13 @@ check_all_running() { ((goterror++)) fi - # check we have the right number of netmon's - how_many_netmons=$(pgrep -a -f ${NETMON_PATH} | wc -l) - if (( ${how_many_running} != ${how_many_netmons} )); then - echo "Wrong number of netmons running (${how_many_running} != ${how_many_netmons}) - stopping" - ((goterror++)) + if [ "$KATA_HYPERVISOR" == "qemu" ]; then + # check we have the right number of netmon's + how_many_netmons=$(pgrep -a -f ${NETMON_PATH} | wc -l) + if (( ${how_many_running} != ${how_many_netmons} )); then + echo "Wrong number of netmons running (${how_many_running} != ${how_many_netmons}) - stopping" + ((goterror++)) + fi fi # check we have no runtimes running (they should be transient, we should not 'see them') diff --git a/process.go b/process.go index f9dc521c8..2d51829d2 100644 --- a/process.go +++ b/process.go @@ -53,17 +53,25 @@ func processRunning(regexps []string) bool { // HypervisorRunning returns true if the hypervisor is still running, otherwise false func HypervisorRunning(containerID string) bool { - hypervisorPath := KataConfig.Hypervisor[DefaultHypervisor].Path - if hypervisorPath == "" { - log.Fatal("Could not determine if hypervisor is running: hypervisor path is empty") - return false + var typeHypervisor = map[string]string{ + DefaultHypervisor: (".*-name.*" + containerID + ".*-qmp.*unix:.*/" + containerID + "/.*"), + FirecrackerHypervisor: (".*--api-sock.*" + containerID + ".*firecracker.sock.*"), + } + for h, r := range typeHypervisor { + config, ok := KataConfig.Hypervisor[h] + if ok { + return processRunning([]string{config.Path + r}) + } } - hypervisorRegexps := []string{hypervisorPath + ".*-name.*" + containerID + ".*-qmp.*unix:.*/" + containerID + "/.*"} - return processRunning(hypervisorRegexps) + log.Fatal("Could not determine if hypervisor is running") + return false } // ProxyRunning returns true if the proxy is still running, otherwise false func ProxyRunning(containerID string) bool { + if _, ok := KataConfig.Hypervisor[FirecrackerHypervisor]; ok { + return false + } proxyPath := KataConfig.Proxy[DefaultProxy].Path if proxyPath == "" { log.Fatal("Could not determine if proxy is running: proxy path is empty")