Support flex volume driver configuration via secret (#113)

Author: Harvey Lowndes

README.md

@@ -16,13 +16,13 @@ installed on every node in your Kubernetes cluster.
 
 ### Kubernetes DaemonSet Installer
 
-The recommended way to install the driver is through the daemonset installer mechanism.
+The recommended way to install the driver is through the daemonset installer mechanism. This will create two daemonsets, one specifically for master nodes, allowing configuration via a Kubernetes Secret, and one for worker nodes.
 
 ```
 kubectl apply -f https://github.com/oracle/oci-flexvolume-driver/releases/download/${flexvolume_driver_version}/oci-flexvolume-driver.yaml
 ```
 
-You'll need to add the config file as per below.
+You'll still need to add the config file manually or as a kubernetes secret.
 
 ### Manually
 
@@ -62,6 +62,18 @@ auth:
 If `"region"` and/or `"compartment"` are not specified in the config file
 they will be retrieved from the hosts [OCI metadata service][4].
 
+### Submit configuration as a Kubernetes secret
+
+The configuration file above can be submitted as a Kubernetes Secret onto the master nodes.
+
+```
+kubectl create secret generic oci-flexvolume-driver \
+    -n kube-system \
+    --from-file=config.yaml=config.yaml
+```
+
+Once the Secret is set and the daemonsets deployed, the configuration file will be placed onto the master nodes.
+
 ##### Using instance principals
 
 To authenticate using [instance principals][9] the following policies must first be 

deploy.sh

@@ -24,13 +24,22 @@ driver_dir="/flexmnt/$VENDOR${VENDOR:+"~"}${DRIVER}"
 
 LOG_FILE="$driver_dir/oci_flexvolume_driver.log"
 
+config_file_name="config.yaml"
+config_tmp_dir="/tmp"
+
+CONFIG_FILE="$config_tmp_dir/$config_file_name"
+
 if [ ! -d "$driver_dir" ]; then
   mkdir "$driver_dir"
 fi
 
 cp "/$DRIVER" "$driver_dir/.$DRIVER"
 mv -f "$driver_dir/.$DRIVER" "$driver_dir/$DRIVER"
 
+if [ -f "$CONFIG_FILE" ]; then
+  cp  "$CONFIG_FILE"  "$driver_dir/$config_file_name"
+fi
+
 while : ; do
   touch $LOG_FILE
   tail -f $LOG_FILE

manifests/oci-flexvolume-driver.yaml

@@ -1,20 +1,60 @@
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
-  name: oci-flexvolume-driver
+  name: oci-flexvolume-driver-master
   namespace: kube-system
 spec:
   template:
     metadata:
-      name: oci-flexvolume-driver
+      name: oci-flexvolume-driver-master
       labels:
         app: oci-flexvolume-driver
     spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: "true"
       tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
       containers:
         - image: iad.ocir.io/__DOCKER_REGISTRY_USERNAME__/oci-flexvolume-driver:__VERSION__
+          imagePullPolicy: Always
+          name: oci-flexvolume-driver
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /flexmnt
+              name: flexvolume-mount
+            - mountPath: /tmp
+              name: config
+              readOnly: true
+      volumes:
+        - name: flexvolume-mount
+          hostPath:
+            path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/
+            type: DirectoryOrCreate
+        - name: config
+          secret:
+            secretName: oci-flexvolume-driver
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: oci-flexvolume-driver-worker
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      name: oci-flexvolume-driver-worker
+      labels:
+        app: oci-flexvolume-driver
+    spec:
+      containers:
+        - image: iad.ocir.io/__DOCKER_REGISTRY_USERNAME__/oci-flexvolume-driver:__VERSION__
+          imagePullPolicy: Always
           name: oci-flexvolume-driver
           securityContext:
             privileged: true

test/system/runner.py

@@ -40,7 +40,8 @@
 MAX_NUM_LOCKFILE_RETRIES = 100
 CI_LOCKFILE_PREFIX = "CI"
 LOCAL_LOCKFILE_PREFIX = "LOCAL"
-DAEMONSET_NAME = "oci-flexvolume-driver"
+WORKER_DAEMONSET_NAME = "oci-flexvolume-driver-worker"
+MASTER_DAEMONSET_NAME = "oci-flexvolume-driver-master"
 CI_APPLICATION_NAME = "oci-flexvolume-driver"
 CI_BASE_URL = "https://app.wercker.com/api/v3"
 CI_PIPELINE_NAME = "system-test"
@@ -355,29 +356,30 @@ def _create_replication_controller_yaml(using_oci, volume_name, test_id):
             volume_name, test_id)
 
 
-def _is_driver_running():
-    stdout = _kubectl("-n kube-system get daemonset " + DAEMONSET_NAME + " -o json", log_stdout=False)
+def _is_driver_running(name):
+    stdout = _kubectl("-n kube-system get daemonset " + name + " -o json", log_stdout=False)
     jsn = json.loads(stdout)
     desired = int(jsn["status"]["desiredNumberScheduled"])
     ready = int(jsn["status"]["numberReady"])
-    _log("    - daemonset " + DAEMONSET_NAME + ": desired: " + str(desired) + ", ready: " + str(ready))
+    _log("    - daemonset " + name + ": desired: " + str(desired) + ", ready: " + str(ready))
     return desired == ready
 
 
-def _wait_for_driver():
+def _wait_for_driver(name):
     num_polls = 0
-    while not _is_driver_running():
+    while not _is_driver_running(name):
         time.sleep(1)
         num_polls += 1
         if num_polls == TIMEOUT:
-            _log("Error: Daemonset: " + DAEMONSET_NAME + " " + "failed to achieve running status: ")
+            _log("Error: Daemonset: " + name + " " + "failed to achieve running status: ")
             _finish_with_exit_code(1)
 
 
 def _install_driver():
     _kubectl("delete -f ../../dist/oci-flexvolume-driver.yaml", exit_on_error=False, display_errors=False)
     _kubectl("apply -f ../../dist/oci-flexvolume-driver.yaml")
-    _wait_for_driver()
+    _wait_for_driver(WORKER_DAEMONSET_NAME)
+    _wait_for_driver(MASTER_DAEMONSET_NAME)
 
 
 def _get_pod_infos(test_id):