Merge pull request #1254 from GabyCT/topic/final1.4

backport stable 1.4

.ci/aarch64/lib_install_qemu_aarch64.sh

@@ -8,6 +8,8 @@ set -e
 
 CURRENT_QEMU_VERSION=$(get_version "assets.hypervisor.qemu.version")
 PACKAGED_QEMU="qemu"
+CURRENT_QEMU_PATCHES_BRANCH=$(get_version "assets.hypervisor.qemu.architecture.aarch64.branch")
+CURRENT_QEMU_COMMIT=$(get_version "assets.hypervisor.qemu.architecture.aarch64.commit")
 
 get_packaged_qemu_version() {
         if [ "$ID" == "ubuntu" ]; then
@@ -54,7 +56,20 @@ build_and_install_qemu() {
 
         pushd "${GOPATH}/src/${QEMU_REPO}"
         git fetch
-        git checkout "$CURRENT_QEMU_VERSION"
+        # if extra patches exist
+        if [ -n "${CURRENT_QEMU_COMMIT}" ]; then
+            git checkout "$CURRENT_QEMU_PATCHES_BRANCH"
+            git checkout "$CURRENT_QEMU_COMMIT"
+            # Apply required patches
+            QEMU_PATCHES_PATH="${GOPATH}/src/${PACKAGING_REPO}/obs-packaging/qemu-aarch64/patches"
+            for patch in ${QEMU_PATCHES_PATH}/*.patch; do
+                echo "Applying patch: $patch"
+                patch -p1 <"$patch"
+            done
+        else
+            git checkout "$CURRENT_QEMU_VERSION"
+        fi
+
         [ -d "capstone" ] || git clone https://github.com/qemu/capstone.git capstone
         [ -d "ui/keycodemapdb" ] || git clone  https://github.com/qemu/keycodemapdb.git ui/keycodemapdb
 
@@ -65,8 +80,11 @@ build_and_install_qemu() {
         echo "Install Qemu"
         sudo -E make install
 
-        # Add link from /usr/local/bin to /usr/bin
-        sudo ln -sf $(command -v qemu-system-${QEMU_ARCH}) "/usr/bin/qemu-system-${QEMU_ARCH}"
+        local qemu_bin=$(command -v qemu-system-${QEMU_ARCH})
+        if [ $(dirname ${qemu_bin}) == "/usr/local/bin" ]; then
+            # Add link from /usr/local/bin to /usr/bin
+            sudo ln -sf $(command -v qemu-system-${QEMU_ARCH}) "/usr/bin/qemu-system-${QEMU_ARCH}"
+        fi
         popd
 }
 

.ci/install_qemu.sh

@@ -101,10 +101,10 @@ main() {
 		else
 			build_and_install_qemu
 		fi
-	elif [ "$QEMU_ARCH" == "aarch64" ]; then
+	elif [ "$QEMU_ARCH" == "aarch64" ] || [ "$QEMU_ARCH" == "ppc64le" ]; then
 		packaged_qemu_version=$(get_packaged_qemu_version)
 		short_current_qemu_version=${CURRENT_QEMU_VERSION#*-}
-		if [ "$packaged_qemu_version" == "$short_current_qemu_version" ]; then
+		if [ "$packaged_qemu_version" == "$short_current_qemu_version" ] && [ -z "${CURRENT_QEMU_COMMIT}" ]; then
 			install_packaged_qemu || build_and_install_qemu
 		else
 			build_and_install_qemu

.ci/jenkins_job_build.sh

@@ -54,7 +54,7 @@ mkdir -p $(dirname "${kata_repo_dir}")
 if [ "${BAREMETAL}" == true ]; then
 	arch=$("${tests_repo_dir}/.ci/kata-arch.sh")
 	echo "Looking for baremetal cleanup script for arch ${arch}"
-	clean_up_script="${tests_repo_dir}/.ci/${arch}/clean_up_${arch}.sh"
+	clean_up_script=("${tests_repo_dir}/.ci/${arch}/clean_up_${arch}.sh") || true
 	if [ -f "${clean_up_script}" ]; then
 		echo "Running baremetal cleanup script for arch ${arch}"
 		tests_repo="${tests_repo}" "${clean_up_script}"

.ci/lib.sh

@@ -198,8 +198,9 @@ kill_stale_process()
 	extract_kata_env
 	stale_process_union=( "${stale_process_union[@]}" "${PROXY_PATH}" "${HYPERVISOR_PATH}" "${SHIM_PATH}" )
 	for stale_process in "${stale_process_union[@]}"; do
-		if pgrep -f "${stale_process}"; then
-			sudo killall -9 "${stale_process}" || true
+		local pids=$(pgrep -d ' ' -f "${stale_process}")
+		if [ -n "$pids" ]; then
+			sudo kill -9 ${pids} || true
 		fi
 	done
 }

.ci/ppc64le/lib_install_qemu_ppc64le.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 IBM Limited
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+CURRENT_QEMU_VERSION=$(get_version "assets.hypervisor.qemu.version")
+PACKAGED_QEMU="qemu-system-ppc"
+BUILT_QEMU="qemu-system-ppc64"
+
+get_packaged_qemu_version() {
+        if [ "$ID" == "ubuntu" ]; then
+		#output redirected to /dev/null
+		sudo apt-get update > /dev/null
+                qemu_version=$(apt-cache madison $PACKAGED_QEMU \
+                        | awk '{print $3}' | cut -d':' -f2 | cut -d'+' -f1 | head -n 1 )
+        elif [ "$ID" == "fedora" ]; then
+                qemu_version=$(sudo dnf --showduplicate list ${PACKAGED_QEMU}.${QEMU_ARCH} \
+                        | awk '/'$PACKAGED_QEMU'/ {print $2}' | cut -d':' -f2 | cut -d'-' -f1 | head -n 1)
+		qemu_version=${qemu_version%.*}
+	elif [ "$ID" == "centos" ]; then
+                qemu_version=$(sudo dnf --showduplicate list ${PACKAGED_QEMU}.${QEMU_ARCH} \
+                        | awk '/'$PACKAGED_QEMU'/ {print $2}' | cut -d':' -f2 | cut -d'-' -f1 | head -n 1)
+        fi
+
+        if [ -z "$qemu_version" ]; then
+                die "unknown qemu version"
+        else
+                echo "${qemu_version}"
+        fi
+}
+
+install_packaged_qemu() {
+        if [ "$ID"  == "ubuntu" ]; then
+                sudo apt install -y "$PACKAGED_QEMU"
+        elif [ "$ID"  == "fedora" ]; then
+                sudo dnf install -y "$PACKAGED_QEMU"
+	elif [ "$ID" == "centos" ]; then
+		sudo yum install -y "$PACKAGED_QEMU"
+        else
+                die "Unrecognized distro"
+        fi
+}
+
+build_and_install_qemu() {
+        QEMU_REPO=$(get_version "assets.hypervisor.qemu.url")
+        # Remove 'https://' from the repo url to be able to clone the repo using 'go get'
+        QEMU_REPO=${QEMU_REPO/https:\/\//}
+        PACKAGING_REPO="github.com/kata-containers/packaging"
+        QEMU_CONFIG_SCRIPT="${GOPATH}/src/${PACKAGING_REPO}/scripts/configure-hypervisor.sh"
+
+        go get -d "${QEMU_REPO}" || true
+        go get -d "$PACKAGING_REPO" || true
+
+        pushd "${GOPATH}/src/${QEMU_REPO}"
+        git fetch
+        git checkout "$CURRENT_QEMU_VERSION"
+        [ -d "capstone" ] || git clone https://github.com/qemu/capstone.git capstone
+        [ -d "ui/keycodemapdb" ] || git clone  https://github.com/qemu/keycodemapdb.git ui/keycodemapdb
+
+        echo "Build Qemu"
+        "${QEMU_CONFIG_SCRIPT}" "qemu" | xargs ./configure
+        make -j $(nproc)
+
+        echo "Install Qemu"
+        sudo -E make install
+
+        sudo ln -sf $(command -v ${BUILT_QEMU}) "/usr/bin/qemu-system-${QEMU_ARCH}"
+        popd
+}

.ci/resolve-kata-dependencies.sh

@@ -13,6 +13,7 @@ proxy_repo="${proxy_repo:-github.com/kata-containers/proxy}"
 runtime_repo="${runtime_repo:-github.com/kata-containers/runtime}"
 shim_repo="${shim_repo:-github.com/kata-containers/shim}"
 tests_repo="${tests_repo:-github.com/kata-containers/tests}"
+packaging_repo="${packaging_repo:-github.com/kata-containers/packaging}"
 
 apply_depends_on() {
 	# kata_repo variable is set by the jenkins_job_build.sh
@@ -58,7 +59,7 @@ apply_depends_on() {
 }
 
 clone_repos() {
-	local kata_repos=( "${agent_repo}" "${proxy_repo}" "${runtime_repo}" "${shim_repo}" "${tests_repo}" )
+	local kata_repos=( "${agent_repo}" "${proxy_repo}" "${runtime_repo}" "${shim_repo}" "${tests_repo}" "${packaging_repo}" )
 	for repo in "${kata_repos[@]}"
 	do
 		echo "Cloning ${repo}"
@@ -89,8 +90,13 @@ clone_repos() {
 			echo "... and rebasing with origin/${branch}"
 			git rebase "origin/${branch}"
 		else
-			echo "Checking out to ${branch}"
-			git fetch origin && git checkout "$branch"
+			# Packaging repo only has master branch, so we
+			# cannot checkout to a different branch.
+			if  [ "${repo}" != "${packaging_repo}" ]
+			then
+				echo "Checking out to ${branch}"
+				git fetch origin && git checkout "$branch"
+			fi
 		fi
 		popd
 	done

.ci/static-checks.sh

@@ -360,6 +360,7 @@ check_license_headers()
 		--exclude="*.png" \
 		--exclude="*.pub" \
 		--exclude="*.service" \
+		--exclude="*.svg" \
 		--exclude="*.toml" \
 		--exclude="*.txt" \
 		--exclude="*.yaml" \
@@ -472,7 +473,7 @@ check_docs()
 		if [ "$specific_branch" != "true" ]
 		then
 			# If the URL is new on this PR, it cannot be checked.
-			echo "$new_urls" | grep -q "\<${url}\>" && \
+			echo "$new_urls" | egrep -q "\<${url}\>" && \
 				info "ignoring new (but correct) URL: $url" && continue
 		fi
 
@@ -540,7 +541,7 @@ check_files()
 
 	info "Checking files"
 
-	if [ "$specifc_branch" = "true" ]
+	if [ "$specific_branch" = "true" ]
 	then
 		info "Checking all files in $branch branch"
 

.ci/teardown.sh

@@ -236,22 +236,11 @@ check_log_files()
 
 		# Display *all* errors caused by runtime exceptions and fatal
 		# signals.
-		for pattern in "fatal error" "fatal signal"
+		for pattern in "fatal error" "fatal signal" "segfault at [0-9]"
 		do
 			# Search for pattern and print all subsequent lines with specified log
 			# level.
-			results=$(sed -ne "/\<${pattern}\>/,\$ p" "$log" || true | grep "level=\"*error\"*")
-			if [ -n "$results" ]
-			then
-				errors=1
-				echo >&2 -e "ERROR: detected ${pattern} in '${log}'\n${results}"
-			fi
-		done
-
-		for pattern in "segfault at [0-9]"
-		do
-			results=$(sed -ne "/\<${pattern}\>/,\$ p" "$log" || true)
-
+			results=$(grep "${pattern}" "$log" || true )
 			if [ -n "$results" ]
 			then
 				errors=1

.travis.yml

@@ -28,7 +28,7 @@ env:
 before_install:
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get update -qq              ; fi
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y -qq automake ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install bash; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then travis_retry brew install bash; fi
 
 script:
   - bash .ci/static-checks.sh github.com/kata-containers/tests

CODEOWNERS

@@ -0,0 +1,13 @@
+# Copyright 2019 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Define any code owners for this repository.
+# The code owners lists are used to help automatically enforce
+# reviews and acks of the right groups on the right PRs.
+
+# Order in this file is important. Only the last match will be
+# used. See https://help.github.com/articles/about-code-owners/
+
+*.md    @kata-containers/documentation
+

Makefile

@@ -8,7 +8,7 @@
 TIMEOUT := 60
 
 # union for 'make test'
-UNION := functional docker crio docker-compose network netmon docker-stability openshift kubernetes swarm vm-factory entropy ramdisk
+UNION := functional docker crio docker-compose network netmon docker-stability oci openshift kubernetes swarm vm-factory entropy ramdisk
 
 # skipped test suites for docker integration tests
 SKIP :=
@@ -63,7 +63,7 @@ docker-compose:
 docker-stability:
 	systemctl is-active --quiet docker || sudo systemctl start docker
 	cd integration/stability && \
-	export ITERATIONS=2 && export MAX_CONTAINERS=20 && chronic ./soak_parallel_rm.sh
+	export ITERATIONS=2 && export MAX_CONTAINERS=20 && ./soak_parallel_rm.sh
 
 kubernetes:
 	bash -f .ci/install_bats.sh
@@ -81,6 +81,11 @@ cri-containerd:
 log-parser:
 	make -C cmd/log-parser
 
+oci:
+	systemctl is-active --quiet docker || sudo systemctl start docker
+	cd integration/oci_calls && \
+	bash -f oci_call_test.sh
+
 openshift:
 	bash -f .ci/install_bats.sh
 	bash -f integration/openshift/run_openshift_tests.sh
@@ -128,6 +133,7 @@ check: checkcommits log-parser
 	ginkgo \
 	kubernetes \
 	log-parser \
+	oci \
 	openshift \
 	pentest \
 	swarm \

bundle.go

@@ -60,6 +60,9 @@ func NewBundle(workload []string) (*Bundle, error) {
 		return nil, err
 	}
 
+	// By default, let's not create a terminal
+	config.Process.Terminal = false
+
 	config.Process.Args = workload
 
 	bundle := &Bundle{

container.go

@@ -185,6 +185,22 @@ func (c *Container) Exec(process Process) (string, string, int) {
 	return cmd.Run()
 }
 
+// State returns the state of the container
+// calls into state command returning its stdout, stderr and exit code
+func (c *Container) State() (string, string, int) {
+	args := []string{}
+
+	args = append(args, "state")
+
+	if c.ID != nil {
+		args = append(args, *c.ID)
+	}
+
+	cmd := NewCommand(Runtime, args...)
+
+	return cmd.Run()
+}
+
 // List the containers
 // calls to list command returning its stdout, stderr and exit code
 func (c *Container) List(format string, quiet bool, all bool) (string, string, int) {

functional/state_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package functional
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/kata-containers/tests"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/extensions/table"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	stateWorkload = []string{"true"}
+)
+
+const (
+	stateStopped  = "stopped"
+	stateWaitTime = 5
+)
+
+var _ = Describe("state", func() {
+	var (
+		container *Container
+		err       error
+	)
+
+	BeforeEach(func() {
+		container, err = NewContainer(stateWorkload, true)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(container).NotTo(BeNil())
+	})
+
+	AfterEach(func() {
+		Expect(container.Teardown()).To(Succeed())
+	})
+
+	DescribeTable("container",
+		func(status string, waitTime int) {
+			_, stderr, exitCode := container.Run()
+			Expect(exitCode).To(Equal(0))
+			Expect(stderr).To(BeEmpty())
+
+			time.Sleep(time.Second * time.Duration(waitTime))
+
+			stdout, stderr, exitCode := container.State()
+			Expect(exitCode).To(Equal(0))
+			Expect(stderr).To(BeEmpty())
+			subString := fmt.Sprintf("\"status\": \"%s\"", status)
+			Expect(stdout).To(ContainSubstring(subString))
+		},
+		Entry(fmt.Sprintf("with workload %s, timeWait %d", stateWorkload, stateWaitTime), stateStopped, stateWaitTime),
+	)
+})