Enhanced self certificates security on Kubernetes deployment

Signed-off-by: Zhizhen Tang <[email protected]>

Author: TangZhiZhen

cdn-server/nginx.conf

@@ -52,9 +52,9 @@ http {
         client_body_timeout 5s;
         client_header_timeout 5s;
 
-        ssl_certificate /run/secrets/self.crt;
-        ssl_certificate_key /run/secrets/self.key;
-        ssl_dhparam /run/secrets/dhparam.pem;
+        ssl_certificate /var/run/secrets/self.crt;
+        ssl_certificate_key /var/run/secrets/self.key;
+        ssl_dhparam /var/run/secrets/dhparam.pem;
 
         ssl_protocols TLSv1.2;
         ssl_prefer_server_ciphers on;

cdn-server/self-sign.sh

@@ -1,11 +1,11 @@
 #!/bin/bash -e
 
-if [ ! -d "/run/secrets" ];then
-    mkdir -p /run/secrets
+if [ ! -d "/var/run/secrets" ];then
+    mkdir -p /var/run/secrets
 fi
 
-openssl req -x509 -nodes -days 30 -newkey rsa:4096 -keyout /run/secrets/self.key -out /run/secrets/self.crt << EOL
-SH
+openssl req -x509 -nodes -days 30 -newkey rsa:4096 -keyout /var/run/secrets/self.key -out /var/run/secrets/self.crt << EOL
+CN
 SH
 Shanghai
 Zizhu
@@ -14,7 +14,7 @@ Intel Corporation
 $1
 [email protected]
 EOL
-chmod 640 "/run/secrets/self.key"
-chmod 644 "/run/secrets/self.crt"
-openssl dhparam -dsaparam -out /run/secrets/dhparam.pem 4096
-chmod 644 "/run/secrets/dhparam.pem"
+chmod 640 "/var/run/secrets/self.key"
+chmod 644 "/var/run/secrets/self.crt"
+openssl dhparam -dsaparam -out /var/run/secrets/dhparam.pem 4096
+chmod 644 "/var/run/secrets/dhparam.pem"

deployment/docker-swarm/docker-compose.yml

@@ -56,17 +56,17 @@ services:
             replicas: 1
         secrets:
             - source: self_crt
-              target: self.crt
+              target: /var/run/secrets/self.crt
               uid: ${USER_ID}
               gid: ${GROUP_ID}
               mode: 0444
             - source: self_key
-              target: self.key
+              target: /var/run/secrets/self.key
               uid: ${USER_ID}
               gid: ${GROUP_ID}
               mode: 0440
             - source: dhparam_pem
-              target: dhparam.pem
+              target: /var/run/secrets/dhparam.pem
               uid: ${USER_ID}
               gid: ${GROUP_ID}
               mode: 0444

deployment/kubernetes/docker-compose-template.yml

@@ -67,7 +67,7 @@ services:
             placement:
                 constraints:
                     - node.hostname == master.machine
-        command: ["bash", "-c", "/home/main.py&/home/self-sign.sh&&/usr/sbin/nginx"]
+        command: ["bash", "-c", "/home/main.py&/usr/sbin/nginx"]
         labels:
             kompose.service.type: NodePort
             kompose.image-pull-policy: IfNotPresent 

deployment/kubernetes/start.sh

@@ -47,6 +47,10 @@ for i in $(find "$DIR" -name "*deployment.yaml"); do
     done
 done
 
+if [ -f "$DIR/ovc-self-certificates.yaml" ]; then
+    kubectl delete -f "$DIR/ovc-self-certificates.yaml"
+fi
+
 rm -rf $DIR/$EXT
 
 yml="$DIR/docker-compose-template.yml"
@@ -66,6 +70,10 @@ try_command kompose convert -f "$yml" -o "$DIR"
 
 try_command "$DIR/update_yaml.py" "$DIR"
 
+try_command kubectl create secret generic ssl-key-secret --from-file=self.key="$DIR/../../self-certificates/self.key" --from-file=self.crt="$DIR/../../self-certificates/self.crt" --from-file=dhparam.pem="$DIR/../../self-certificates/dhparam.pem" --dry-run -o yaml > "$DIR/ovc-self-certificates.yaml"
+
+try_command kubectl apply -f "$DIR/ovc-self-certificates.yaml"
+
 for i in $(find "$DIR" -path "$DIR/dashboard" -prune -o -type f -name "*service.yaml" -print); do
     kubectl apply -f "$i"
 done

deployment/kubernetes/stop.sh

@@ -47,4 +47,8 @@ for i in $(find "$DIR" -name "*deployment.yaml"); do
     done
 done
 
+if [ -f "$DIR/ovc-self-certificates.yaml" ]; then
+    kubectl delete -f "$DIR/ovc-self-certificates.yaml"
+fi
+
 rm -rf $DIR/$EXT

deployment/kubernetes/yaml_utils.py

@@ -56,6 +56,9 @@ def add_volumeMounts(data, isCDN):
                                'readOnly': False},
                               {'name': 'html',
                                'mountPath': '/var/www/html',
+                               'readOnly': True},
+                              {'name': 'secrets',
+                               'mountPath': '/var/run/secrets',
                                'readOnly': True} ]
     else:
         volumemounts_caps = [ {'name': 'archive',
@@ -93,7 +96,9 @@ def add_volumes(data, nfs_server, isCDN, cdn_directory):
                           {'path': cdn_directory + '/volume/video/hls'} },
                          {'name': 'html',
                           'hostPath':
-                          {'path': cdn_directory + '/volume/html'} } ]
+                          {'path': cdn_directory + '/volume/html'} },
+                         {'name': 'secrets',
+                          'secret': {'secretName': 'ssl-key-secret'} } ]
     elif isCDN:
         volumes_caps = [ {'name': 'archive',
                           'nfs':
@@ -109,7 +114,9 @@ def add_volumes(data, nfs_server, isCDN, cdn_directory):
                            'server': nfs_server} },
                          {'name': 'html',
                           'hostPath':
-                          {'path': cdn_directory + '/volume/html'} } ]
+                          {'path': cdn_directory + '/volume/html'} },
+                         {'name': 'secrets',
+                          'secret': {'secretName': 'ssl-key-secret'} } ]
     else:
         volumes_caps = [ {'name': 'archive',
                           'nfs':

script/Kubernetes_setup_ubuntu_master.sh

@@ -40,6 +40,10 @@ fi
 
 # Install packages
 # Set Proxy if need
+proxy_http=$http_proxy
+proxy_https=$https_proxy
+export http_proxy=$proxy_http
+export https_proxy=$proxy_https
 try_command apt-get update && apt-get install -y apt-transport-https curl
 try_command curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 try_command cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
@@ -75,6 +79,8 @@ try_command systemctl restart docker
 try_command systemctl restart kubelet
 
 # Kubeadm init
+unset http_proxy
+unset https_proxy
 try_command kubeadm init --pod-network-cidr=10.244.0.0/16
 try_command mkdir -p $HOME/.kube
 try_command cp -f /etc/kubernetes/admin.conf $HOME/.kube/config
@@ -83,5 +89,7 @@ try_command export KUBECONFIG=$HOME/.kube/config
 try_command kubectl taint nodes --all node-role.kubernetes.io/master-
 
 # Set Proxy if need
+export http_proxy=$proxy_http
+export https_proxy=$proxy_https
 try_command kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml
 try_command sed -i '/- kube-apiserver/a\\    - --service-node-port-range=1-65535' /etc/kubernetes/manifests/kube-apiserver.yaml