Skip to content
This repository was archived by the owner on Jun 28, 2024. It is now read-only.

Commit 70e8394

Browse files
author
Julio Montes
authored
Merge pull request #983 from chavafg/topic/k8s-runtimeclass
k8s: Add runtimeclass configuration
2 parents 46398e0 + 49e5e1e commit 70e8394

29 files changed

Lines changed: 425 additions & 18 deletions

‎.ci/configure_containerd_for_kata.sh‎

100644100755
File mode changed.

‎.ci/configure_crio_for_kata.sh‎

100644100755
Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,22 @@ set -o pipefail
1111

1212
crio_config_file="/etc/crio/crio.conf"
1313

14-
echo "Configure runtimes for trusted/untrusted annotations"
15-
sudo sed -i 's/^#* *runtime =.*/runtime = "\/usr\/local\/bin\/crio-runc"/' "$crio_config_file"
16-
sudo sed -i 's/^default_runtime/# default_runtime/' "$crio_config_file"
17-
sudo sed -i 's/^#*runtime_untrusted_workload = ""/runtime_untrusted_workload = "\/usr\/local\/bin\/kata-runtime"/' "$crio_config_file"
18-
sudo sed -i 's/#*default_workload_trust = ""/default_workload_trust = "trusted"/' "$crio_config_file"
19-
14+
# `use_runtime_class` should be set to:
15+
# - true if we will test using k8s RuntimeClass feature or
16+
# - false (default) if we will test using the old trusted/untrusted annotations.
17+
use_runtime_class=${use_runtime_class:-false}
2018

19+
if [ "${use_runtime_class}" == true ]; then
20+
echo "Configure runtimes map for RuntimeClass feature"
21+
echo "- Set runc as default runtime"
22+
sudo sed -i 's!runtime_path =.*!runtime_path = "/usr/local/bin/crio-runc"!' "$crio_config_file"
23+
echo "- Add kata-runtime to the runtimes map"
24+
sudo sed -i '/crio-runc/a[crio.runtime.runtimes.kata]' "$crio_config_file"
25+
sudo sed -i '/kata/aruntime_path = "/usr/local/bin/kata-runtime"' "$crio_config_file"
26+
else
27+
echo "Configure runtimes for trusted/untrusted annotations"
28+
sudo sed -i 's!^#* *runtime =.*!runtime = "/usr/local/bin/crio-runc"!' "$crio_config_file"
29+
sudo sed -i 's!^default_runtime!# default_runtime!' "$crio_config_file"
30+
sudo sed -i 's!^#*runtime_untrusted_workload = ""!runtime_untrusted_workload = "/usr/local/bin/kata-runtime"!' "$crio_config_file"
31+
sudo sed -i 's!#*default_workload_trust = ""!default_workload_trust = "trusted"!' "$crio_config_file"
32+
fi

‎.ci/install_kubernetes.sh‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,18 @@
55
# SPDX-License-Identifier: Apache-2.0
66
#
77

8-
set -e
8+
set -o errexit
9+
set -o nounset
10+
set -o pipefail
911

1012
cidir=$(dirname "$0")
1113
source "${cidir}/lib.sh"
1214

15+
# `use_runtime_class` should be set to:
16+
# - true if we will test using k8s RuntimeClass feature or
17+
# - false (default) if we will test using the old trusted/untrusted annotations.
18+
use_runtime_class=${use_runtime_class:-false}
19+
1320
echo "Install Kubernetes components"
1421

1522
cidir=$(dirname "$0")
@@ -27,3 +34,10 @@ EOF"
2734
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
2835
chronic sudo -E apt update
2936
chronic sudo -E apt install --allow-downgrades -y kubelet="$kubernetes_version" kubeadm="$kubernetes_version" kubectl="$kubernetes_version"
37+
38+
if [ "${use_runtime_class}" == true ]; then
39+
kubelet_systemd_file="/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
40+
feature_gate="--feature-gates RuntimeClass=true"
41+
echo "Configure Kubelet service to enable RuntimeClass feature"
42+
sudo sed -i "s/ExecStart=\/.*$/& $feature_gate/" "$kubelet_systemd_file"
43+
fi

‎integration/kubernetes/init.sh‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,18 @@
55
# SPDX-License-Identifier: Apache-2.0
66
#
77

8-
set -e
8+
set -o errexit
9+
set -o nounset
10+
set -o pipefail
11+
912

1013
SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
1114
source "${SCRIPT_PATH}/../../.ci/lib.sh"
1215
source "${SCRIPT_PATH}/../../lib/common.bash"
1316

1417
cri_runtime="${CRI_RUNTIME:-crio}"
18+
use_runtime_class=${use_runtime_class:-false}
19+
kubernetes_version=$(get_version "externals.kubernetes.version")
1520

1621
case "${cri_runtime}" in
1722
containerd)
@@ -46,6 +51,12 @@ kubeadm_config_file="$(mktemp --tmpdir kubeadm_config.XXXXXX.yaml)"
4651

4752
sed -e "s|CRI_RUNTIME_SOCKET|${cri_runtime_socket}|" "${kubeadm_config_template}" > "${kubeadm_config_file}"
4853

54+
if [ "${use_runtime_class}" == true ]; then
55+
echo "Add RuntimeClass feature for apiserver in kubeadm config file"
56+
echo "apiServerExtraArgs:" >> "${kubeadm_config_file}"
57+
echo " feature-gates: RuntimeClass=true" >> "${kubeadm_config_file}"
58+
fi
59+
4960
sudo -E kubeadm init --config "${kubeadm_config_file}"
5061

5162
export KUBECONFIG=/etc/kubernetes/admin.conf
@@ -66,5 +77,14 @@ sleep_time=5
6677
cmd="sudo -E kubectl get pods --all-namespaces | grep 'coredns.*1/1.*Running'"
6778
waitForProcess "$dns_wait_time" "$sleep_time" "$cmd"
6879

80+
if [ "${use_runtime_class}" == true ]; then
81+
runtimeclass_files_path="${SCRIPT_PATH}/runtimeclass_workloads"
82+
echo "Install RuntimeClass resource definition"
83+
sudo -E kubectl apply -f \
84+
"https://raw.githubusercontent.com/kubernetes/kubernetes/v${kubernetes_version/-*}/cluster/addons/runtimeclass/runtimeclass_crd.yaml"
85+
echo "Create kata RuntimeClass resource"
86+
sudo -E kubectl create -f "${runtimeclass_files_path}/kata-runtimeclass.yaml"
87+
fi
88+
6989
# Enable the master node to be able to schedule pods.
7090
sudo -E kubectl taint nodes "$(hostname)" node-role.kubernetes.io/master:NoSchedule-

‎integration/kubernetes/k8s-attach-handlers.bats‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@ load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh"
1010
setup() {
1111
export KUBECONFIG=/etc/kubernetes/admin.conf
1212
pod_name="handlers"
13-
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
13+
14+
if sudo -E kubectl get runtimeclass | grep kata; then
15+
pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
16+
else
17+
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
18+
fi
1419
}
1520

1621
@test "Running with postStart and preStop handlers" {

‎integration/kubernetes/k8s-cpu-ns.bats‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,12 @@ setup() {
1717
total_cpus=2
1818
total_requests=512
1919
total_cpu_container=1
20-
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
20+
21+
if sudo -E kubectl get runtimeclass | grep kata; then
22+
pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
23+
else
24+
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
25+
fi
2126
}
2227

2328
@test "Check CPU constraints" {

‎integration/kubernetes/k8s-env.bats‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@ load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh"
1010
setup() {
1111
export KUBECONFIG=/etc/kubernetes/admin.conf
1212
pod_name="test-env"
13-
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
13+
14+
if sudo -E kubectl get runtimeclass | grep kata; then
15+
pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
16+
else
17+
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
18+
fi
1419
}
1520

1621
@test "Environment variables" {

‎integration/kubernetes/k8s-liveness-probes.bats‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@ load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh"
1010
setup() {
1111
export KUBECONFIG=/etc/kubernetes/admin.conf
1212
pod_name="liveness-exec"
13-
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
13+
14+
if sudo -E kubectl get runtimeclass | grep kata; then
15+
pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
16+
else
17+
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
18+
fi
1419
}
1520

1621
@test "Liveness probe" {

‎integration/kubernetes/k8s-memory.bats‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@ load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh"
1010
setup() {
1111
export KUBECONFIG=/etc/kubernetes/admin.conf
1212
pod_name="memory-test"
13-
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
13+
14+
if sudo -E kubectl get runtimeclass | grep kata; then
15+
pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
16+
else
17+
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
18+
fi
1419
}
1520

1621
@test "Exceeding memory constraints" {

‎integration/kubernetes/k8s-pid-ns.bats‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,12 @@ setup() {
1313
pod_name="busybox"
1414
first_container_name="first-test-container"
1515
second_container_name="second-test-container"
16-
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
16+
17+
if sudo -E kubectl get runtimeclass | grep kata; then
18+
pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
19+
else
20+
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
21+
fi
1722
}
1823

1924
@test "Check PID namespaces" {

0 commit comments

Comments
 (0)