From 5adf576baa966ef9b355525ccedd3ccf02abe475 Mon Sep 17 00:00:00 2001
From: wangfei <wf01031309@alibaba-inc.com>
Date: Fri, 18 Nov 2022 10:17:48 +0800
Subject: [PATCH] update scripts

---
 context/rootfs/scripts/clean-kube.sh        |  33 -----
 context/rootfs/scripts/docker.sh            | 116 +++++++++++++++
 context/rootfs/scripts/init-kube.sh         |  72 ++++-----
 context/rootfs/scripts/init-registry.sh     | 153 ++++++--------------
 context/rootfs/scripts/init.sh              |  16 +-
 context/rootfs/scripts/install-cri.sh       | 115 ---------------
 context/rootfs/scripts/kubelet-pre-start.sh |  78 +++++++---
 context/rootfs/scripts/nvidia-docker.sh     | 117 +++++++++++++++
 context/rootfs/scripts/uninstall-docker.sh  |  36 +++++
 context/rootfs/scripts/utils.sh             |  96 ++++++++++++
 10 files changed, 496 insertions(+), 336 deletions(-)
 delete mode 100644 context/rootfs/scripts/clean-kube.sh
 create mode 100644 context/rootfs/scripts/docker.sh
 delete mode 100644 context/rootfs/scripts/install-cri.sh
 create mode 100644 context/rootfs/scripts/nvidia-docker.sh
 create mode 100644 context/rootfs/scripts/uninstall-docker.sh
 create mode 100644 context/rootfs/scripts/utils.sh

diff --git a/context/rootfs/scripts/clean-kube.sh b/context/rootfs/scripts/clean-kube.sh
deleted file mode 100644
index c612548..0000000
--- a/context/rootfs/scripts/clean-kube.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Copyright © 2021 Alibaba Group Holding Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-systemctl disable kubelet
-rm -f /usr/bin/conntrack
-rm -f /usr/bin/kubelet-pre-start.sh
-rm -f /usr/bin/crictl
-rm -f /usr/bin/kubeadm
-rm -f /usr/bin/kubectl
-rm -f /usr/bin/kubelet
-rm -f /usr/bin/containerd-rootless-setuptool.sh
-rm -f /usr/bin/containerd-rootless.sh
-rm -f /usr/bin/nerdctl
-rm -f /usr/bin/seautil
-
-rm -f /etc/sysctl.d/k8s.conf
-rm -f /etc/systemd/system/kubelet.service
-rm -rf /etc/systemd/system/kubelet.service.d
-rm -rf /var/lib/kubelet/
-rm -f /var/lib/kubelet/config.yaml
-systemctl daemon-reload
diff --git a/context/rootfs/scripts/docker.sh b/context/rootfs/scripts/docker.sh
new file mode 100644
index 0000000..c3aba19
--- /dev/null
+++ b/context/rootfs/scripts/docker.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+# Copyright © 2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+
+scripts_path=$(cd `dirname $0`; pwd)
+image_dir="$scripts_path/../images"
+DOCKER_VERSION="19.03.14-sealer"
+
+get_distribution() {
+  lsb_dist=""
+  # Every system that we officially support has /etc/os-release
+  if [ -r /etc/os-release ]; then
+    lsb_dist="$(. /etc/os-release && echo "$ID")"
+  fi
+  # Returning an empty string here should be alright since the
+  # case statements don't act unless you provide an actual value
+  echo "$lsb_dist"
+}
+
+disable_selinux() {
+  if [ -s /etc/selinux/config ] && grep 'SELINUX=enforcing' /etc/selinux/config; then
+    sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
+    setenforce 0
+  fi
+}
+
+load_images() {
+  for image in "$image_dir"/*; do
+    if [ -f "${image}" ]; then
+      docker load -q -i "${image}"
+    fi
+  done
+}
+
+check_docker_valid() {
+  if ! docker info 2>&1; then
+    panic "docker is not healthy: $(docker info 2>&1), please check"
+  fi
+
+  dockerVersion=`docker info --format '{{json .ServerVersion}}' | tr -d '"'`
+  if [ "${dockerVersion}" != "${DOCKER_VERSION}" ]; then
+    panic "docker version is ${dockerVersion}, should be 19.03.15, please check"
+  fi
+}
+
+storage=${1:-/var/lib/docker}
+mkdir -p $storage
+if ! utils_command_exists docker; then
+  lsb_dist=$(get_distribution)
+  lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"
+  echo "current system is $lsb_dist"
+  case "$lsb_dist" in
+  ubuntu | deepin | debian | raspbian)
+    cp "${scripts_path}"/../etc/docker.service /lib/systemd/system/docker.service
+    if [ ! -f /usr/sbin/iptables ];then
+      if [ -f /sbin/iptables ];then
+        ln -s /sbin/iptables /usr/sbin/iptables
+      else
+        panic "iptables not found, please check"
+      fi
+    fi
+    ;;
+  centos | rhel | anolis | ol | sles | kylin | neokylin)
+    cp "${scripts_path}"/../etc/docker.service /usr/lib/systemd/system/docker.service
+    ;;
+  alios)
+    docker0=$(ip addr show docker0 | head -1|tr " " "\n"|grep "<"|grep -iwo "UP"|wc -l)
+    if [ "$docker0" != "1" ]; then
+        ip link add name docker0 type bridge
+        ip addr add dev docker0 172.17.0.1/16
+    fi
+    cp "${scripts_path}"/../etc/docker.service /usr/lib/systemd/system/docker.service
+    ;;
+  *)
+    utils_info "unknown system to use /lib/systemd/system/docker.service"
+    cp "${scripts_path}"/../etc/docker.service /lib/systemd/system/docker.service
+    ;;
+  esac
+
+  [ -d /etc/docker/ ] || mkdir /etc/docker/ -p
+
+  chmod -R 755 "${scripts_path}"/../cri
+  tar -zxvf "${scripts_path}"/../cri/docker.tar.gz -C /usr/bin
+  chmod a+x /usr/bin
+  chmod a+x /usr/bin/docker
+  chmod a+x /usr/bin/dockerd
+  systemctl enable docker.service
+  systemctl restart docker.service
+  cp "${scripts_path}"/../etc/daemon.json /etc/docker
+  mkdir -p /root/.docker/
+  cp "${scripts_path}"/../etc/docker-cli-config.json /root/.docker/config.json
+  if [[ -n $1 && -n $2 ]]; then
+    sed -i "s/sea.hub:5000/$2:$3/g" /etc/docker/daemon.json
+  fi
+fi
+
+disable_selinux
+systemctl daemon-reload
+systemctl restart docker.service
+check_docker_valid
+
+load_images
\ No newline at end of file
diff --git a/context/rootfs/scripts/init-kube.sh b/context/rootfs/scripts/init-kube.sh
index a2a1800..5e9c2d1 100644
--- a/context/rootfs/scripts/init-kube.sh
+++ b/context/rootfs/scripts/init-kube.sh
@@ -1,26 +1,9 @@
 #!/bin/bash
-# shellcheck disable=SC1091
 
-# Open ipvs
-modprobe -- ip_vs
-modprobe -- ip_vs_rr
-modprobe -- ip_vs_wrr
-modprobe -- ip_vs_sh
-modprobe -- br_netfilter
-## version_ge 4.19 4.19 true ;
-## version_ge 5.4 4.19 true ;
-## version_ge 3.10 4.19 false ;
+scripts_path=$(cd `dirname $0`; pwd)
+source "${scripts_path}"/utils.sh
 
-version_ge() {
-  test "$(echo "$@" | tr ' ' '\n' | sort -rV | head -n 1)" == "$1"
-}
-
-disable_selinux() {
-  if [ -s /etc/selinux/config ] && grep 'SELINUX=enforcing' /etc/selinux/config; then
-    sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
-    setenforce 0
-  fi
-}
+set -x
 
 get_distribution() {
   lsb_dist=""
@@ -50,32 +33,27 @@ disable_firewalld() {
   esac
 }
 
-kernel_version=$(uname -r | cut -d- -f1)
-if version_ge "${kernel_version}" 4.19; then
-  modprobe -- nf_conntrack
-else
-  modprobe -- nf_conntrack_ipv4
-fi
-
-cat <<EOF >/etc/sysctl.d/k8s.conf
-net.bridge.bridge-nf-call-ip6tables = 1
-net.bridge.bridge-nf-call-iptables = 1
-net.ipv4.conf.all.rp_filter=0
-EOF
-sysctl --system
-sysctl -w net.ipv4.ip_forward=1
-disable_firewalld
-swapoff -a || true
-disable_selinux
+copy_bins() {
+  chmod -R 755 ../bin/*
+  chmod 644 ../bin
+  cp ../bin/* /usr/bin
+  cp ../scripts/kubelet-pre-start.sh /usr/bin
+  chmod +x /usr/bin/kubelet-pre-start.sh
+}
 
-chmod -R 755 ../bin/*
-chmod 644 ../bin
-cp ../bin/* /usr/bin
-cp ../scripts/kubelet-pre-start.sh /usr/bin
-# Cgroup driver
-mkdir -p /etc/systemd/system
-cp ../etc/kubelet.service /etc/systemd/system/
-[ -d /etc/systemd/system/kubelet.service.d ] || mkdir /etc/systemd/system/kubelet.service.d
-cp ../etc/10-kubeadm.conf /etc/systemd/system/kubelet.service.d/
+copy_kubelet_service(){
+  mkdir -p /etc/systemd/system
+  cp ../etc/kubelet.service /etc/systemd/system/
+  [ -d /etc/systemd/system/kubelet.service.d ] || mkdir /etc/systemd/system/kubelet.service.d
+  cp ../etc/10-kubeadm.conf /etc/systemd/system/kubelet.service.d/
+}
 
-systemctl daemon-reload && systemctl enable kubelet
+disable_firewalld
+copy_bins
+copy_kubelet_service
+[ -d /var/lib/kubelet ] || mkdir -p /var/lib/kubelet/
+/usr/bin/kubelet-pre-start.sh
+systemctl enable kubelet
+
+# nvidia-docker.sh need set kubelet labels, it should be run after kubelet
+bash ${scripts_path}/nvidia-docker.sh || exit 1
\ No newline at end of file
diff --git a/context/rootfs/scripts/init-registry.sh b/context/rootfs/scripts/init-registry.sh
index 3c40221..a9918a6 100644
--- a/context/rootfs/scripts/init-registry.sh
+++ b/context/rootfs/scripts/init-registry.sh
@@ -1,44 +1,14 @@
 #!/bin/bash
-# Copyright © 2021 Alibaba Group Holding Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 set -e
 set -x
 # prepare registry storage as directory
-# shellcheck disable=SC2046
 cd $(dirname "$0")
 
-# shellcheck disable=SC2034
 REGISTRY_PORT=${1-5000}
 VOLUME=${2-/var/lib/registry}
 REGISTRY_DOMAIN=${3-sea.hub}
 
-# shellcheck disable=SC2068
-utils_command_exists() {
-  command -v $@ >/dev/null 2>&1
-}
-
-CONTAINER_RUNTIME=docker
-if ! utils_command_exists docker; then
-  if containerd --version; then
-    CONTAINER_RUNTIME=containerd
-  else
-    echo "either docker and containerd not found, please check" >&2
-    exit 1
-  fi
-fi
-
 container=sealer-registry
 rootfs=$(dirname "$(pwd)")
 config="$rootfs/etc/registry_config.yml"
@@ -48,86 +18,49 @@ image_dir="$rootfs/images"
 
 mkdir -p "$VOLUME" || true
 
-# shellcheck disable=SC2068
-runtimeRun() {
-  if [ "$CONTAINER_RUNTIME" == "containerd" ]; then
-    nerdctl container run $@
-  else
-    docker run $@
-  fi
-}
-
-# shellcheck disable=SC2068
-runtimeStart() {
-  if [ "$CONTAINER_RUNTIME" == "containerd" ]; then
-    nerdctl start $@
-  else
-    docker start $@
-  fi
-}
-
-# shellcheck disable=SC2068
-runtimeInspect() {
-  if [ "$CONTAINER_RUNTIME" == "containerd" ]; then
-    nerdctl container inspect $@
-  else
-    docker inspect $@
-  fi
-}
-
-# shellcheck disable=SC2068
-runtimeGetContainerStatus() {
-  if [ "$CONTAINER_RUNTIME" == "containerd" ]; then
-    nerdctl container inspect $@ | grep '"Status"' | awk '{print $2}' | tr -d ','
-  else
-    docker inspect --format '{{json .State.Status}}' $@
-  fi
-}
-
 startRegistry() {
-  n=1
-  while ((n <= 3)); do
-    echo "attempt to start registry"
-    runtimeStart $container && break
-    ((n++))
-    sleep 3
-  done
+    n=1
+    while (( n <= 3 ))
+    do
+        echo "attempt to start registry"
+        (docker start $container && break) || (( n < 3))
+        (( n++ ))
+        sleep 3
+    done
 }
 
 load_images() {
-  for image in "$image_dir"/*; do
-    if [ ! -f "${image}" ]; then
-      continue
-    fi
-    if [ "$CONTAINER_RUNTIME" == "containerd" ]; then
-      ctr image import "${image}"
-    else
-      docker load -q -i "${image}"
-    fi
-  done
+for image in "$image_dir"/*
+do
+ if [ -f "${image}" ]
+ then
+  docker load -q -i "${image}"
+ fi
+done
 }
 
 check_registry() {
-  n=1
-  while ((n <= 3)); do
-    registry_status=$(runtimeGetContainerStatus sealer-registry)
-    [[ "$registry_status" == \"running\" ]] && break
-    ((n++))
-    sleep 3
-  done
-  if [[ "$registry_status" != \"running\" ]]; then
-    echo "sealer-registry is not running, status: $registry_status" >&2
-    exit 1
-  fi
+    n=1
+    while (( n <= 3 ))
+    do
+        registry_status=$(docker inspect --format '{{json .State.Status}}' sealer-registry)
+        if [[ "$registry_status" == \"running\" ]]; then
+            break
+        fi
+        if [[ $n -eq 3 ]]; then
+           echo "sealer-registry is not running, status: $registry_status"
+           exit 1
+        fi
+        (( n++ ))
+        sleep 3
+    done
 }
 
 load_images
 
 ## rm container if exist.
-if [ "$CONTAINER_RUNTIME" == "containerd" ]; then
-  runtimeInspect $container &>/dev/null && nerdctl rm -f $container
-else
-  docker inspect $container &>/dev/null && docker rm -f $container
+if [ "$(docker ps -aq -f name=$container)" ]; then
+    docker rm -f $container
 fi
 
 regArgs="-d --restart=always \
@@ -138,22 +71,20 @@ regArgs="-d --restart=always \
 -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/$REGISTRY_DOMAIN.crt \
 -e REGISTRY_HTTP_TLS_KEY=/certs/$REGISTRY_DOMAIN.key"
 
-if [ -f "$config" ]; then
-  sed -i "s/5000/$1/g" "$config"
-  regArgs="$regArgs \
+if [ -f $config ]; then
+    sed -i "s/5000/$1/g" $config
+    regArgs="$regArgs \
     -v $config:/etc/docker/registry/config.yml"
 fi
 
-if [ -f "$htpasswd" ]; then
-  runtimeRun "$regArgs" \
-    -v "$htpasswd":/htpasswd \
-    -e REGISTRY_AUTH=htpasswd \
-    -e REGISTRY_AUTH_HTPASSWD_PATH=/htpasswd \
-    -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" registry:2.7.1 || startRegistry
+if [ -f $htpasswd ]; then
+    docker run $regArgs \
+            -v $htpasswd:/htpasswd \
+            -e REGISTRY_AUTH=htpasswd \
+            -e REGISTRY_AUTH_HTPASSWD_PATH=/htpasswd \
+            -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" registry:2.7.1 || startRegistry
 else
-  runtimeRun "$regArgs" registry:2.7.1 || startRegistry
+    docker run $regArgs registry:2.7.1 || startRegistry
 fi
 
-if ! check_registry; then
-  exit 1
-fi
+check_registry
\ No newline at end of file
diff --git a/context/rootfs/scripts/init.sh b/context/rootfs/scripts/init.sh
index 6971936..0fc4c1d 100644
--- a/context/rootfs/scripts/init.sh
+++ b/context/rootfs/scripts/init.sh
@@ -13,20 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
-set -x
-
-#STORAGE=${1:-/var/lib/docker} will delete
+STORAGE=${1:-/var/lib/docker}
 REGISTRY_DOMAIN=${2-sea.hub}
 REGISTRY_PORT=${3-5000}
 
-chmod -R 755 ../bin/*
-chmod 644 ../bin
-cp ../bin/* /usr/bin
-
-chmod a+x install-cri.sh
-
-./install-cri.sh "$STORAGE" "$REGISTRY_DOMAIN" "$REGISTRY_PORT"
+# Install docker
+chmod a+x docker.sh
+#./docker.sh  /var/docker/lib  sealer.hub 5001
+bash docker.sh ${STORAGE} ${REGISTRY_DOMAIN} $REGISTRY_PORT
 
 chmod a+x init-kube.sh
 
diff --git a/context/rootfs/scripts/install-cri.sh b/context/rootfs/scripts/install-cri.sh
deleted file mode 100644
index fe81f8f..0000000
--- a/context/rootfs/scripts/install-cri.sh
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/bin/bash
-# shellcheck disable=SC1091
-# Copyright © 2021 Alibaba Group Holding Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -x
-set -e
-
-rootfs=$(dirname "$(pwd)")
-image_dir="$rootfs/images"
-lib_dir="${rootfs}/lib"
-dump_config_dir="$rootfs/etc/dump-config.toml"
-
-command_exists() {
-  command -v "$@" >/dev/null 2>&1
-}
-get_distribution() {
-  lsb_dist=""
-  # Every system that we officially support has /etc/os-release
-  if [ -r /etc/os-release ]; then
-    lsb_dist="$(. /etc/os-release && echo "$ID")"
-  fi
-  # Returning an empty string here should be alright since the
-  # case statements don't act unless you provide an actual value
-  echo "$lsb_dist"
-}
-disable_selinux() {
-  if [ -s /etc/selinux/config ] && grep 'SELINUX=enforcing' /etc/selinux/config; then
-    sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
-    setenforce 0
-  fi
-}
-server_load_images() {
-  for image in "$image_dir"/*; do
-    if [ -f "${image}" ]; then
-      ${1} load -i "${image}"
-    fi
-  done
-}
-
-##cri is docker
-if [[ $(ls ../cri/docker*.tar.gz) ]]; then
-  if ! command_exists docker; then
-    lsb_dist=$(get_distribution)
-    lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"
-    echo "current system is $lsb_dist"
-    case "$lsb_dist" in
-    ubuntu | deepin | debian | raspbian)
-      cp ../etc/docker.service /lib/systemd/system/docker.service
-      ;;
-    centos | rhel | ol | sles | kylin | neokylin)
-      cp ../etc/docker.service /usr/lib/systemd/system/docker.service
-      ;;
-    alios)
-      ip link add name docker0 type bridge
-      ip addr add dev docker0 172.17.0.1/16
-      cp ../etc/docker.service /usr/lib/systemd/system/docker.service
-      ;;
-    *)
-      echo "unknown system to use /lib/systemd/system/docker.service"
-      cp ../etc/docker.service /lib/systemd/system/docker.service
-      ;;
-    esac
-
-    [ -d /etc/docker/ ] || mkdir /etc/docker/ -p
-
-    chmod -R 755 ../cri
-    tar -zxvf ../cri/docker*.tar.gz -C /usr/bin
-    chmod a+x /usr/bin
-    chmod a+x /usr/bin/docker
-    chmod a+x /usr/bin/dockerd
-    systemctl enable docker.service
-    systemctl restart docker.service
-    cp ../etc/daemon.json /etc/docker
-    if [[ -n $2 && -n $3 ]]; then
-      sed -i "s/sea.hub:5000/$2:$3/g" /etc/docker/daemon.json
-    fi
-  fi
-  disable_selinux
-  systemctl daemon-reload
-  systemctl enable docker.service
-  systemctl restart docker.service
-  load_image_server="docker"
-else
-  if ! command_exists containerd; then
-    tar zxvf ../cri/cri-*.tar.gz -C /
-    cd "$lib_dir" && source install_libseccomp.sh
-  fi
-  systemctl daemon-reload
-  systemctl enable containerd.service
-  systemctl restart containerd.service
-
-  sed -i "s/sea.hub/${2:-sea.hub}/g" "$dump_config_dir"
-  sed -i "s/5000/${3:-5000}/g" "$dump_config_dir"
-
-  #add cri sandbox image and sea.hub registry cert path
-  ##sandbox_image = "sea.hub:5000/pause:3.6" custom setup
-  mkdir -p /etc/containerd
-  containerd --config "$dump_config_dir" config dump >/etc/containerd/config.toml
-  systemctl restart containerd.service
-  load_image_server="nerdctl"
-fi
-
-server_load_images "${load_image_server}"
diff --git a/context/rootfs/scripts/kubelet-pre-start.sh b/context/rootfs/scripts/kubelet-pre-start.sh
index d899967..7a75092 100644
--- a/context/rootfs/scripts/kubelet-pre-start.sh
+++ b/context/rootfs/scripts/kubelet-pre-start.sh
@@ -1,35 +1,75 @@
 #!/bin/bash
-# Open ipvs
-modprobe -- ip_vs
-modprobe -- ip_vs_rr
-modprobe -- ip_vs_wrr
-modprobe -- ip_vs_sh
-modprobe -- br_netfilter
+# this file can't import utils.sh, cause it will be put into /usr/bin for kubelet.service
+
 version_ge() {
   test "$(echo "$@" | tr ' ' '\n' | sort -rV | head -n 1)" == "$1"
 }
+
 disable_selinux() {
   if [ -s /etc/selinux/config ] && grep 'SELINUX=enforcing' /etc/selinux/config; then
     sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
     setenforce 0
   fi
+  if ! getenforce | grep Disabled;then
+    setenforce 0 || true
+  fi
 }
 
-kernel_version=$(uname -r | cut -d- -f1)
-if version_ge "${kernel_version}" 4.19; then
-  modprobe -- nf_conntrack
-else
-  modprobe -- nf_conntrack_ipv4
-fi
+set_modules() {
+  if [ ! -d /etc/modprobe.d/ ]; then
+    echo "we can't find dir /etc/sysconfig/modules/, so linux mod can't be reloaded after reboot, please check"
+    exit 1
+  fi
+
+  # put modprobe configuration into ackdistro.modules
+  modfile=/etc/modprobe.d/ackdistro.modules
+  cat <<EOF >${modfile}
+modprobe -- ip_vs
+modprobe -- ip_vs_rr
+modprobe -- ip_vs_wrr
+modprobe -- ip_vs_sh
+modprobe -- br_netfilter
+modprobe -- xt_set
+modprobe -- ip_tables
+modprobe -- ip6_tables
+EOF
+
+  kernel_version=$(uname -r | cut -d- -f1)
+  if version_ge "${kernel_version}" 4.19; then
+    echo "modprobe -- nf_conntrack" >>${modfile}
+  else
+    echo "modprobe -- nf_conntrack_ipv4" >>${modfile}
+  fi
+
+  chmod 755 /etc/modprobe.d/ackdistro.modules
+  /etc/modprobe.d/ackdistro.modules
+}
 
-cat <<EOF >/etc/sysctl.d/k8s.conf
+set_sysctl() {
+  cat <<EOF >/etc/sysctl.d/k8s.conf
+# set by ack-distro
 net.bridge.bridge-nf-call-ip6tables = 1
 net.bridge.bridge-nf-call-iptables = 1
-net.ipv4.conf.all.rp_filter=0
+net.ipv4.ip_forward = 1
+net.ipv4.tcp_tw_reuse = 1
+net.ipv4.tcp_fin_timeout = 15
+net.ipv6.conf.all.forwarding = 1
+net.ipv4.conf.all.arp_filter = 0
+net.ipv4.conf.all.rp_filter = 0
 EOF
-sysctl --system
-sysctl -w net.ipv4.ip_forward=1
-# systemctl stop firewalld && systemctl disable firewalld
-swapoff -a
+
+  while read -r line;do
+    if ! grep "${line}" /etc/sysctl.conf;then
+      echo "${line}" >> /etc/sysctl.conf
+    fi
+  done < /etc/sysctl.d/k8s.conf
+
+  sysctl --system
+}
+
+swapoff -a || true
+[[ -f /etc/fstab ]] && sed -i '/\sswap\s/d' /etc/fstab
+iptables -P FORWARD ACCEPT
 disable_selinux
-exit 0
+set_modules
+set_sysctl
\ No newline at end of file
diff --git a/context/rootfs/scripts/nvidia-docker.sh b/context/rootfs/scripts/nvidia-docker.sh
new file mode 100644
index 0000000..b1be5ee
--- /dev/null
+++ b/context/rootfs/scripts/nvidia-docker.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+
+scripts_path=$(cd `dirname $0`; pwd)
+source "${scripts_path}"/utils.sh
+
+set -x
+
+# NVIDIA_VERSION=v1.0.1
+GPU_FOUNDED=0
+
+# Check if customer buys gpu capablities inaglity
+GPU_SUPPORT=0
+
+RPM_DIR=${scripts_path}/../rpm/nvidia
+
+public::nvidia::check(){
+    if [ "$ARCH" != "amd64" ];then
+        utils_info "gpu now not support $ARCH"
+        return
+    fi
+    if which nvidia-smi;then
+        GPU_SUPPORT=1
+    fi
+}
+
+public::nvidia::enable_gpu_capability(){
+    utils_arch_env
+    public::nvidia::check
+    if [[ "0" == "$GPU_SUPPORT" ]]; then
+        return
+    fi
+
+    kube::nvidia::detect_gpu
+    if [[ "1" == "$GPU_FOUNDED" ]]; then
+        public::nvidia::install_nvidia_docker2
+    fi
+}
+
+public::nvidia::enable_gpu_device_plugin() {
+    if [[ "0" == "$GPU_SUPPORT" ]] || [[ "0" == "$GPU_FOUNDED" ]]; then
+        return
+    fi
+
+    sleep 10
+    public::nvidia::deploy_static_pod
+}
+
+kube::nvidia::detect_gpu(){
+    tar -xvf ${scripts_path}/../tgz/nvidia.tgz -C ${scripts_path}/../rpm/
+    kube::nvidia::setup_lspci
+    lspci | grep -i nvidia > /dev/null 2>&1
+    if [[ "$?" == "0" ]]; then
+        export GPU_FOUNDED=1
+    fi
+}
+
+kube::nvidia::setup_lspci(){
+    if utils_command_exists lspci; then
+        return
+    fi
+    utils_info "lspci command not exist, install it"
+    rpm -ivh --force --nodeps ${RPM_DIR}/pciutils*.rpm
+    if [[ "$?" != "0" ]]; then
+        panic "failed to install pciutils via command (rpm -ivh --force --nodeps ${RPM_DIR}/pciutils*.rpm) in dir ${PWD}, please run it for debug"
+    fi
+}
+
+
+public::nvidia::install_nvidia_driver(){
+    # see cos/release in branch agility-develop for details. Installing driver is not supported in trident.
+    utils_info 'installing nvidia driver is not supported.'
+    return
+}
+
+
+public::nvidia::install_nvidia_docker2(){
+    sleep 3
+    if  `which nvidia-container-runtime > /dev/null 2>&1` && [ $(echo $((docker info | grep nvidia) | wc -l)) -gt 1 ] ; then
+        utils_info 'nvidia-container-runtime is already insatlled'
+        return
+    fi
+
+    # 1. Install nvidia-container-runtime
+    if ! output=$(rpm -ivh --force --nodeps `ls ${RPM_DIR}/*.rpm` 2>&1);then
+        panic "failed to install rpm, output:${output}, maybe your rpm db was broken, please see https://cloudlinux.zendesk.com/hc/en-us/articles/115004075294-Fix-rpmdb-Thread-died-in-Berkeley-DB-library for help"
+    fi
+
+    # 2. Update docker daemon.json and reload docker daemon
+    if [[ -f /etc/docker/daemon.json.rpmorig ]];then
+        mv -f /etc/docker/daemon.json.rpmorig /etc/docker/daemon.json
+    fi
+
+    mkdir -p /etc/docker
+    sed -i '2 i\
+    \"default-runtime\": \"nvidia\",\
+    \"runtimes\": {\
+        \"nvidia\": {\
+            \"path\": \"/usr/bin/nvidia-container-runtime\",\
+            \"runtimeArgs\": []\
+        }\
+    },' /etc/docker/daemon.json
+
+    # To do: we need make sure if it's better to reload rather than restart, e.g. service docker restart
+    pkill -SIGHUP dockerd
+    utils_info 'nvidia-docker2 installed'
+}
+
+# deploy nvidia plugin in static pod
+public::nvidia::deploy_static_pod() {
+    mkdir -p /etc/kubernetes/manifests
+    cp -f ${scripts_path}/../statics/nvidia-device-plugin.yml /etc/kubernetes/manifests/nvidia-device-plugin.yml
+
+    utils_info "nvidia-device-plugin yaml succefully deployed ..."
+}
+
+public::nvidia::enable_gpu_capability
+public::nvidia::enable_gpu_device_plugin
\ No newline at end of file
diff --git a/context/rootfs/scripts/uninstall-docker.sh b/context/rootfs/scripts/uninstall-docker.sh
new file mode 100644
index 0000000..76bb408
--- /dev/null
+++ b/context/rootfs/scripts/uninstall-docker.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+systemctl stop docker
+ip link delete docker0 type bridge || true
+rm -rf /lib/systemd/system/docker.service
+rm -rf /usr/lib/systemd/system/docker.service
+rm -rf /etc/docker/daemon.json
+systemctl daemon-reload
+
+rm -f /usr/bin/conntrack
+rm -f /usr/bin/kubelet-pre-start.sh
+rm -f /usr/bin/containerd
+rm -f /usr/bin/containerd-shim
+rm -f /usr/bin/containerd-shim-runc-v2
+rm -f /usr/bin/crictl
+rm -f /usr/bin/ctr
+rm -f /usr/bin/docker
+rm -f /usr/bin/docker-init
+rm -f /usr/bin/docker-proxy
+rm -f /usr/bin/dockerd
+rm -f /usr/bin/kubeadm
+rm -f /usr/bin/kubectl
+rm -f /usr/bin/kubelet
+rm -f /usr/bin/rootlesskit
+rm -f /usr/bin/rootlesskit-docker-proxy
+rm -f /usr/bin/runc
+rm -f /usr/bin/vpnkit
+rm -f /usr/bin/containerd-rootless-setuptool.sh
+rm -f /usr/bin/containerd-rootless.sh
+rm -f /usr/bin/nerdctl
+
+rm -f /etc/sysctl.d/k8s.conf
+rm -f /etc/systemd/system/kubelet.service
+rm -rf /etc/systemd/system/kubelet.service.d
+rm -rf /var/lib/kubelet/
+rm -f /var/lib/kubelet/config.yaml
\ No newline at end of file
diff --git a/context/rootfs/scripts/utils.sh b/context/rootfs/scripts/utils.sh
new file mode 100644
index 0000000..1ee33ac
--- /dev/null
+++ b/context/rootfs/scripts/utils.sh
@@ -0,0 +1,96 @@
+#!/bin/bash
+
+utils_version_ge() {
+  test "$(echo "$@" | tr ' ' '\n' | sort -rV | head -n 1)" == "$1"
+}
+
+# This function will display the message in red, and exit immediately.
+panic()
+{
+    set +x
+    echo -e "\033[1;31mPanic error: $@, please check this panic\033[0m"
+    exit 1
+    set -x
+}
+
+utils_info()
+{
+    echo -e "\033[1;32m$@\033[0m"
+}
+
+utils_command_exists() {
+    command -v "$@" > /dev/null 2>&1
+}
+
+utils_arch_env() {
+    ARCH=$(uname -m)
+    case $ARCH in
+        armv5*) ARCH="armv5" ;;
+        armv6*) ARCH="armv6" ;;
+        armv7*) ARCH="armv7" ;;
+        aarch64) ARCH="arm64" ;;
+        x86) ARCH="386" ;;
+        x86_64) ARCH="amd64" ;;
+        i686) ARCH="386" ;;
+        i386) ARCH="386" ;;
+    esac
+}
+
+utils_os_env() {
+    ubu=$(cat /etc/issue | grep -i "ubuntu" | wc -l)
+    debian=$(cat /etc/issue | grep -i "debian" | wc -l)
+    cet=$(cat /etc/centos-release | grep "CentOS" | wc -l)
+    redhat=$(cat /etc/redhat-release | grep "Red Hat" | wc -l)
+    alios=$(cat /etc/redhat-release | grep "Alibaba" | wc -l)
+    kylin=$(cat /etc/kylin-release | grep -E "Kylin" | wc -l)
+    anolis=$(cat /etc/anolis-release | grep -E "Anolis" | wc -l)
+    if [ "$ubu" == "1" ];then
+        export OS="Ubuntu"
+    elif [ "$cet" == "1" ];then
+        export OS="CentOS"
+    elif [ "$redhat" == "1" ];then
+        export OS="RedHat"
+    elif [ "$debian" == "1" ];then
+        export OS="Debian"
+    elif [ "$alios" == "1" ];then
+        export OS="AliOS"
+    elif [ "$kylin" == "1" ];then
+        export OS="Kylin"
+    elif [ "$anolis" == 1 ];then
+        export OS="Anolis"
+    else
+       panic "unkown os...   exit"
+    fi
+
+    case "$OS" in
+        CentOS)
+            export OSVersion="$(cat /etc/centos-release | awk '{print $4}')"
+            ;;
+        AliOS)
+            export OSVersion="$(cat /etc/alios-release | awk '{print $7}')"
+            ;;
+        Kylin)
+            export OSVersion="$(cat /etc/kylin-release | awk '{print $6}')"
+            ;;
+        Anolis)
+            export OSVersion="$(cat /etc/anolis-release | awk '{print $4}')"
+            ;;
+        *)
+            echo -e "Not support get OS version of ${OS}"
+    esac
+
+    if [[ "$OS" == "CentOS" ]] || [[ "$OS" == "Anolis" ]] ||  [[ "$OS" == "AliOS" ]];then
+        export OSRelease="el7"
+        # vague compare: 8.x.xxx
+        if [[ $OSVersion =~ ^8\..*$ ]];then
+            export OSRelease="el8"
+        fi
+    fi
+}
+
+utils_shouldMkFs() {
+    if [ "$1" != "" ] && [ "$1" != "/" ] && [ "$1" != "\"/\"" ];then
+        return 0
+    fi
+    return 1
+}
\ No newline at end of file