Merge pull request #926 from chavafg/topic/changes-k8s-1.12

Update ci and tests scripts for kubernetes 1.12

Author: teawater

.ci/install_crio.sh

@@ -5,15 +5,19 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-set -e
+set -o errexit
+set -o nounset
+set -o pipefail
 
 cidir=$(dirname "$0")
 source "${cidir}/lib.sh"
 source /etc/os-release || source /usr/lib/os-release
 
 echo "Get CRI-O sources"
 kubernetes_sigs_org="github.com/kubernetes-sigs"
+ghprbGhRepository="${ghprbGhRepository:-}"
 crio_repo="${kubernetes_sigs_org}/cri-o"
+
 go get -d "$crio_repo" || true
 pushd "${GOPATH}/src/${crio_repo}"
 
@@ -93,41 +97,33 @@ sudo -E install -D -m0755 runc "/usr/local/bin/crio-runc"
 popd
 
 crio_config_file="/etc/crio/crio.conf"
-echo "Set runc as default runtime in CRI-O for trusted workloads"
-sudo sed -i 's/^runtime =.*/runtime = "\/usr\/local\/bin\/crio-runc"/' "$crio_config_file"
 
-echo "Change stream_port where cri-o will listen"
-sudo sed -i 's/^stream_port.*/stream_port = "10020"/' "$crio_config_file"
+echo "Set manage_network_ns_lifecycle to true"
+network_ns_flag="manage_network_ns_lifecycle"
+sudo sed -i "/\[crio.runtime\]/a$network_ns_flag = true" "$crio_config_file"
 
 echo "Add docker.io registry to pull images"
-# Matches cri-o 1.9 file format
-sudo sed -i 's/^registries = \[/registries = \[ "docker.io"/' "$crio_config_file"
 # Matches cri-o 1.10 file format
+sudo sed -i 's/^registries = \[/registries = \[ "docker.io"/' "$crio_config_file"
+# Matches cri-o 1.12 file format
 sudo sed -i 's/^#registries = \[/registries = \[ "docker.io" \] /' "$crio_config_file"
 
-echo "Set manage_network_ns_lifecycle to true"
-network_ns_flag="manage_network_ns_lifecycle"
-
-# Check if flag is already defined in the CRI-O config file.
-# If it is already defined, then just change the value to true,
-# else, add the flag with the value.
-if grep "$network_ns_flag" "$crio_config_file"; then
-	sudo sed -i "s/^$network_ns_flag.*/$network_ns_flag = true/" "$crio_config_file"
-else
-	sudo sed -i "/\[crio.runtime\]/a$network_ns_flag = true" "$crio_config_file"
-fi
+echo "Change stream_port where cri-o will listen"
+sudo sed -i 's/^stream_port.*/stream_port = "10020"/' "$crio_config_file"
 
-echo "Set Kata containers as default runtime in CRI-O for untrusted workloads"
-sudo sed -i 's/default_workload_trust = "trusted"/default_workload_trust = "untrusted"/' "$crio_config_file"
-sudo sed -i 's/runtime_untrusted_workload = ""/runtime_untrusted_workload = "\/usr\/local\/bin\/kata-runtime"/' "$crio_config_file"
+echo "Configure runtimes for trusted/untrusted annotations"
+sudo sed -i 's/^#* *runtime =.*/runtime = "\/usr\/local\/bin\/crio-runc"/' "$crio_config_file"
+sudo sed -i 's/^default_runtime/# default_runtime/' "$crio_config_file"
+sudo sed -i 's/^#*runtime_untrusted_workload = ""/runtime_untrusted_workload = "\/usr\/local\/bin\/kata-runtime"/' "$crio_config_file"
+sudo sed -i 's/#*default_workload_trust = ""/default_workload_trust = "trusted"/' "$crio_config_file"
 
 service_path="/etc/systemd/system"
 crio_service_file="${cidir}/data/crio.service"
 
 echo "Install crio service (${crio_service_file})"
 sudo install -m0444 "${crio_service_file}" "${service_path}"
 
-kubelet_service_dir="/etc/systemd/system/kubelet.service.d/"
+kubelet_service_dir="${service_path}/kubelet.service.d/"
 
 sudo mkdir -p "${kubelet_service_dir}"
 

.ci/lib.sh

@@ -255,5 +255,10 @@ gen_clean_arch() {
 	delete_stale_kata_resource
 	info "Remove installed kata packages"
 	${GOPATH}/src/${tests_repo}/cmd/kata-manager/kata-manager.sh remove-packages
+	info "Remove installed kubernetes packages and configuration"
+	if [ "$ID" == ubuntu ]; then
+		sudo rm -rf /etc/systemd/system/kubelet.service.d
+		sudo apt-get purge kubeadm kubelet kubectl -y
+	fi
 }
 

integration/kubernetes/cleanup_env.sh

@@ -4,31 +4,34 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+# This script is used to reset the kubernetes cluster
 
 SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
 source "${SCRIPT_PATH}/../../lib/common.bash"
 
-export KUBECONFIG=/etc/kubernetes/admin.conf
-sudo -E kubeadm reset --cri-socket=/var/run/crio/crio.sock
+cri_runtime="${CRI_RUNTIME:-crio}"
+
+case "${cri_runtime}" in
+containerd)
+	cri_runtime_socket="/run/containerd/containerd.sock"
+	;;
+crio)
+	cri_runtime_socket="/var/run/crio/crio.sock"
+	;;
+*)
+	echo "Runtime ${cri_runtime} not supported"
+	;;
+esac
 
-# Workaround to delete pods using crictl
-# Needed until https://github.com/kubernetes/kubeadm/issues/748
-# gets fixed
-for ctr in $(sudo crictl ps --quiet); do
-	sudo crictl stop "$ctr"
-	sudo crictl rm "$ctr"
-done
-for pod in $(sudo crictl pods --quiet); do
-	sudo crictl stopp "$pod"
-	sudo crictl rmp "$pod"
-done
+export KUBECONFIG=/etc/kubernetes/admin.conf
+sudo -E kubeadm reset -f --cri-socket="${cri_runtime_socket}"
 
-sudo systemctl stop crio
+sudo systemctl stop "${cri_runtime}"
 
 sudo ip link set dev cni0 down
 sudo ip link set dev flannel.1 down
 sudo ip link del cni0
 sudo ip link del flannel.1
 
-# Check no processes are left behind
+# Check no kata processes are left behind after reseting kubernetes
 check_processes

integration/kubernetes/data/kube-flannel-rbac.yml

@@ -15,27 +15,19 @@ cri_runtime="${CRI_RUNTIME:-crio}"
 
 case "${cri_runtime}" in
 containerd)
-    cri_runtime_socket="/run/containerd/containerd.sock"
-    ;;
+	cri_runtime_socket="/run/containerd/containerd.sock"
+	;;
 crio)
-    cri_runtime_socket="/var/run/crio/crio.sock"
-    ;;
+	cri_runtime_socket="/var/run/crio/crio.sock"
+	;;
 *)
-    echo "Runtime ${cri_runtime} not supported"
-
-    ;;
+	echo "Runtime ${cri_runtime} not supported"
+	;;
 esac
 
-# Check no processes are left behind
+# Check no there are no kata processes from previous tests.
 check_processes
 
-# The next workaround is to be able to communicate between pods
-# Issue: https://github.com/kubernetes/kubernetes/issues/40182
-# Fix is ready for K8s 1.9, but still need to investigate why it does not
-# work by default.
-# FIXME: Issue: https://github.com/clearcontainers/tests/issues/934
-sudo iptables -P FORWARD ACCEPT
-
 # Remove existing CNI configurations:
 sudo rm -rf /var/lib/cni/networks/*
 sudo rm -rf /etc/cni/net.d/*
@@ -49,18 +41,30 @@ echo "Start ${cri_runtime} service"
 sudo systemctl start ${cri_runtime}
 
 echo "Init cluster using ${cri_runtime_socket}"
-sudo -E kubeadm init --pod-network-cidr 10.244.0.0/16 --cri-socket="unix://${cri_runtime_socket}"
+kubeadm_config_template="${SCRIPT_PATH}/kubeadm/config.yaml"
+kubeadm_config_file="$(mktemp --tmpdir kubeadm_config.XXXXXX.yaml)"
+
+sed -e "s|CRI_RUNTIME_SOCKET|${cri_runtime_socket}|" "${kubeadm_config_template}" > "${kubeadm_config_file}"
+
+sudo -E kubeadm init --config "${kubeadm_config_file}"
+
 export KUBECONFIG=/etc/kubernetes/admin.conf
 
 sudo -E kubectl get nodes
 sudo -E kubectl get pods
-sudo -E kubectl create -f "${SCRIPT_PATH}/data/kube-flannel-rbac.yml"
-sudo -E kubectl create --namespace kube-system -f "${SCRIPT_PATH}/data/kube-flannel.yml"
 
-# The kube-dns pod usually takes around 30 seconds to get ready
+# kube-flannel config file taken from k8s 1.12 documentation:
+flannel_config="https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml"
+
+sudo -E kubectl apply -f "$flannel_config"
+
+# The kube-dns pod usually takes around 120 seconds to get ready
 # This instruction will wait until it is up and running, so we can
 # start creating our containers.
-dns_wait_time=300
+dns_wait_time=120
 sleep_time=5
-cmd="sudo -E kubectl get pods --all-namespaces | grep 'dns.*3/3.*Running'"
+cmd="sudo -E kubectl get pods --all-namespaces | grep 'coredns.*1/1.*Running'"
 waitForProcess "$dns_wait_time" "$sleep_time" "$cmd"
+
+# Enable the master node to be able to schedule pods.
+sudo -E kubectl taint nodes "$(hostname)" node-role.kubernetes.io/master:NoSchedule-

integration/kubernetes/data/kube-flannel.yml

@@ -17,20 +17,15 @@ setup() {
 	total_cpus=2
 	total_requests=512
 	total_cpu_container=1
+	pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
 }
 
 @test "Check CPU constraints" {
-	issue="https://github.com/kata-containers/tests/issues/794"
-	[ "${CRI_RUNTIME}" == "containerd" ] && skip "test not working with ${CRI_RUNTIME} see: ${issue}"
-	wait_time=120
-	sleep_time=5
-
 	# Create the pod
-	sudo -E kubectl create -f pod-cpu.yaml
+	sudo -E kubectl create -f "${pod_config_dir}/pod-cpu.yaml"
 
 	# Check pod creation
-	pod_status_cmd="sudo -E kubectl get pods -a | grep $pod_name | grep Running"
-	waitForProcess "$wait_time" "$sleep_time" "$pod_status_cmd"
+	sudo -E kubectl wait --for=condition=Ready pod "$pod_name"
 
 	# Check the total of cpus
 	total_cpus_container=$(sudo -E kubectl exec $pod_name -c $container_name nproc)
@@ -54,9 +49,5 @@ setup() {
 }
 
 teardown() {
-       sudo -E kubectl delete deployment "$pod_name"
-       # Wait for the pods to be deleted
-       cmd="sudo -E kubectl get pods | grep found."
-       waitForProcess "$wait_time" "$sleep_time" "$cmd"
-       sudo -E kubectl get pods
+       sudo -E kubectl delete pod "$pod_name"
 }