Merge pull request #414 from rojkov/fpga-plugin-kustomize

fpga: finalize plugin kustomization

Author: bart0sh

cmd/fpga_admissionwebhook/README.md

@@ -98,7 +98,7 @@ mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgawebhook-mutat
 clusterrole.rbac.authorization.k8s.io/intelfpgawebhook-manager-role created
 clusterrolebinding.rbac.authorization.k8s.io/intelfpgawebhook-manager-rolebinding created
 service/intelfpgawebhook-webhook-service created
-deployment.apps/intelfpgawebhook-controller-manager created
+deployment.apps/intelfpgawebhook-webhook created
 certificate.cert-manager.io/intelfpgawebhook-serving-cert created
 issuer.cert-manager.io/intelfpgawebhook-selfsigned-issuer created
 ```

cmd/fpga_plugin/README.md

@@ -11,17 +11,12 @@
     * [Getting the source code](#getting-the-source-code)
     * [Verify node kubelet config](#verify-node-kubelet-config)
     * [Deploying as a DaemonSet](#deploying-as-a-daemonset)
-        * [Create a service account](#create-a-service-account)
-        * [Deploying `region` mode](#deploying-region-mode)
-        * [Deploying `af` mode](#deploying-af-mode)
-        * [Deploy the DaemonSet](#deploy-the-daemonset)
         * [Verify plugin registration](#verify-plugin-registration)
         * [Building the plugin image](#building-the-plugin-image)
     * [Deploy by hand](#deploy-by-hand)
         * [Build FPGA device plugin](#build-fpga-device-plugin)
         * [Run FPGA device plugin in af mode](#run-fpga-device-plugin-in-af-mode)
         * [Run FPGA device plugin in region mode](#run-fpga-device-plugin-in-region-mode)
-* [Next steps](#next-steps)
 
 # Introduction
 
@@ -177,82 +172,83 @@ $ ls /var/lib/kubelet/device-plugins/kubelet.sock
 
 ## Deploying as a DaemonSet
 
-You can deploy the plugin in either of the modes, using the
-[DaemonSet YAML](../../deployments/fpga_plugin/fpga_plugin.yaml)
-supplied. Details are in the following sections. Actions common to both deployment modes are detailed
-first. Mode specific actions are then detailed.
+As a pre-requisite you need to have [cert-manager](https://cert-manager.io)
+up and running:
 
-If you intend to deploy your own image, you will need to reference the
-[image build section](#build-the-plugin-image) first.
-
-If you do not want to deploy the `devel` tagged image, you will need to edit the
-YAML deployment files to reference your required image.
-
-### For beta testing: new deployment model
-
-The FPGA plugin deployment is currently being rewritten to enable
-straight-forward deployment of both `af` and
-`region` modes. The deployment has two steps:
+```bash
+$ kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.1/cert-manager.yaml
+$ kubectl get pods -n cert-manager
+NAME                                      READY   STATUS    RESTARTS   AGE
+cert-manager-7747db9d88-bd2nl             1/1     Running   0          1m
+cert-manager-cainjector-87c85c6ff-59sb5   1/1     Running   0          1m
+cert-manager-webhook-64dc9fff44-29cfc     1/1     Running   0          1m
 
-1. Run `scripts/fpga-plugin-prepare-for-kustomization.sh`. This will
-   create the necessary secrets: a key and a signed certificate for
-   the FPGA admission controller.
+```
 
-2. Depending on the FPGA mode, run either
-   ```bash
-   $ kubectl create -k deployments/fpga_plugin/overlays/af
-   ```
-   or
-   ```bash
-   $ kubectl create -k deployments/fpga_plugin/overlays/region
-   ```
-   This will create the service account and deploy
-   both the FPGA plugin and the admission controller in the chosen mode.
+Depending on the FPGA mode, run either
+```bash
+$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/fpga_plugin/overlays/af
+namespace/intelfpgaplugin-system created
+customresourcedefinition.apiextensions.k8s.io/acceleratorfunctions.fpga.intel.com created
+customresourcedefinition.apiextensions.k8s.io/fpgaregions.fpga.intel.com created
+mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgaplugin-mutating-webhook-configuration created
+clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-manager-role created
+clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-node-getter created
+clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-get-nodes created
+clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-manager-rolebinding created
+service/intelfpgaplugin-webhook-service created
+deployment.apps/intelfpgaplugin-webhook created
+daemonset.apps/intelfpgaplugin-fpgadeviceplugin created
+certificate.cert-manager.io/intelfpgaplugin-serving-cert created
+issuer.cert-manager.io/intelfpgaplugin-selfsigned-issuer created
+```
+or
+```bash
+$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/fpga_plugin/overlays/region
+namespace/intelfpgaplugin-system created
+customresourcedefinition.apiextensions.k8s.io/acceleratorfunctions.fpga.intel.com created
+customresourcedefinition.apiextensions.k8s.io/fpgaregions.fpga.intel.com created
+mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgaplugin-mutating-webhook-configuration created
+clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-manager-role created
+clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-node-getter created
+clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-get-nodes created
+clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-manager-rolebinding created
+service/intelfpgaplugin-webhook-service created
+deployment.apps/intelfpgaplugin-webhook created
+daemonset.apps/intelfpgaplugin-fpgadeviceplugin created
+certificate.cert-manager.io/intelfpgaplugin-serving-cert created
+issuer.cert-manager.io/intelfpgaplugin-selfsigned-issuer created
+```
+The command should result in two pods running:
+```bash
+$ kubectl get pods -n intelfpgaplugin-system
+NAME                                       READY   STATUS    RESTARTS   AGE
+intelfpgaplugin-fpgadeviceplugin-skcw5     1/1     Running   0          57s
+intelfpgaplugin-webhook-7d6bcb8b57-k52b9   1/1     Running   0          57s
+```
 
-This deployment model is under development. The remaining part of this
-document goes through the current deployment model: here for the
-FPGA plugin and in the next document for the FPGA admission controller.
+If you intend to deploy your own image, you will need to reference the
+[image build section](#build-the-plugin-image) first.
 
-### Create a service account
+If you do not want to deploy the `devel` or release tagged image, you will need to create your
+own kustomization overlay referencing your required image.
 
-To deploy the plugin in a production cluster, create a service account
-for the plugin:
+If you need the FPGA plugin on some nodes to operate in a different mode then add this
+annotation to the nodes:
 
 ```bash
-$ kubectl create -f deployments/fpga_plugin/fpga_plugin_service_account.yaml
-serviceaccount/intel-fpga-plugin-controller created
-clusterrole.rbac.authorization.k8s.io/node-getter created
-clusterrolebinding.rbac.authorization.k8s.io/get-nodes created
+$ kubectl annotate node <node_name> 'fpga.intel.com/device-plugin-mode=region'
 ```
-
-### Deploying `region` mode
-
-To deploy the FPGA plugin DaemonSet in `region` mode, you need to set the plugin
-mode annotation on all of your nodes, otherwise the FPGA plugin will run in its default
-`af` mode.
-
+or
 ```bash
-$ kubectl annotate node --all 'fpga.intel.com/device-plugin-mode=region'
+$ kubectl annotate node <node_name> 'fpga.intel.com/device-plugin-mode=af'
 ```
+And restart the pods on the nodes.
 
-### Deploying `af` mode
-
-To deploy the FPGA plugin DaemonSet in `af` mode, you do not need to set the mode annotation on
-your nodes, as the FPGA plugin runs in `af` mode by default.
-
-> **Note:** The FPGA plugin [DaemonSet YAML](../../deployments/fpga_plugin/fpga_plugin.yaml)
+> **Note:** The FPGA plugin [DaemonSet YAML](../../deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml)
 > also deploys the [FPGA CRI-O hook](../fpga_criohook) `initcontainer` image, but it will be
 > benign (un-used) when running the FPGA plugin in `af` mode.
 
-### Deploy the DaemonSet
-
-You can then use the example DaemonSet YAML file provided to deploy the plugin.
-
-```bash
-$ kubectl create -f deployments/fpga_plugin/fpga_plugin.yaml
-daemonset.apps/intel-fpga-plugin created
-```
-
 ### Verify plugin registration
 
 Verify the FPGA plugin has been deployed on the nodes. The below shows the output
@@ -288,8 +284,8 @@ Successfully tagged intel/intel-fpga-plugin:devel
 
 This image launches `fpga_plugin` in `af` mode by default.
 
-To use your own container image, modify the
-[`deployments/fpga_plugin/fpga_plugin.yaml`](../../deployments/fpga_plugin/fpga_plugin.yaml)
+To use your own container image, create you own kustomization overlay patching
+[`deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml`](../../deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml)
 file.
 
 ## Deploy by hand
@@ -337,6 +333,3 @@ FPGA device plugin started in region mode
 device-plugin start server at: /var/lib/kubelet/device-plugins/fpga.intel.com-region-ce48969398f05f33946d560708be108a.sock
 device-plugin registered
 ```
-# Next steps
-
-Continue installation with the [FPGA admission controller webhook](../fpga_admissionwebhook/README.md).

demo/screencast-fpga-orchestrated.sh

@@ -39,18 +39,12 @@ cleanup()
   out 'Cleanup demo artifacts' 20
   out 'delete test pod:' 20
   command 'kubectl delete pod test-fpga-region || true' 20
-  out 'delete ServiceAccount:' 20
-  command 'kubectl delete ServiceAccount intel-fpga-plugin-controller --namespace kube-system || true' 20
-  out 'delete ClusterRole:' 20
-  command 'kubectl delete ClusterRole node-getter --namespace kube-system || true' 20
-  out 'delete ClusterRoleBinding:' 20
-  command 'kubectl delete ClusterRoleBinding get-nodes --namespace kube-system || true' 20
+  out 'delete mappings' 20
+  command 'kubectl delete -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml || true' 200
+  out 'delete namespace and all the objects in the intelfpgaplugin-system namespace:' 20
+  command 'kubectl delete namespace intelfpgaplugin-system || true' 20
   out 'delete node annotation:' 20
   command 'kubectl annotate node --all fpga.intel.com/device-plugin-mode- || true' 20
-  out 'delete plugin daemonset:' 20
-  command 'kubectl delete daemonset intel-fpga-plugin --namespace kube-system || true' 20
-  out 'delete webhook deployment:' 20
-  command 'kubectl delete deployment intel-fpga-webhook-deployment || true' 20
 }
 
 record()
@@ -83,26 +77,11 @@ screen2()
 }
 
 screen3()
-{
-  clear
-  cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
-  out '3. Deploy admission controller webhook'
-  command 'kubectl apply -k deployments/fpga_admissionwebhook/default'
-  sleep 2
-  out 'Check if its pod is running:'
-  command 'kubectl get pods -n intelfpgawebhook-system'
-  out 'Deploy the mappings:'
-  command 'kubectl apply -f deployments/fpga_admissionwebhook/mappings-collection.yaml'
-  out 'Check pod logs:'
-  command "kubectl logs $(kubectl get pods -n intelfpgawebhook-system| grep intelfpgawebhook-controller-manager | awk '{print $1}') -n intelfpgawebhook-system"
-}
-
-screen4()
 {
   clear
   cd /srv/demo
   sudo rm -rf /srv/intel.com/fpga/Arria10.dcp1.2 /srv/intel.com/fpga/69528db6eb31577a8c3668f9faa081f6
-  out '4. Create bistream storage'
+  out '3. Create bistream storage'
   out 'Create directory for Arria10.dcp1.2 interface id:'
   command 'sudo mkdir -p /srv/intel.com/fpga/69528db6eb31577a8c3668f9faa081f6'
   out 'Create Arria10.dcp1.2 symlink for convenience:'
@@ -116,30 +95,30 @@ screen4()
   command 'ls -la /srv/intel.com/fpga/Arria10.dcp1.2/'
 }
 
-screen5()
+screen4()
 {
   clear
   cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
-  out '5. Deploy FPGA plugin'
-  out 'Create a service account for the plugin'
-  command 'kubectl create -f deployments/fpga_plugin/fpga_plugin_service_account.yaml'
-  out 'Set region mode for the plugin:'
-  command "kubectl annotate node --all 'fpga.intel.com/device-plugin-mode=region'"
-  out 'Create plugin daemonset:'
-  command 'kubectl create -f deployments/fpga_plugin/fpga_plugin.yaml'
-  out 'Check if its pod is runnning:'
-  command 'kubectl get pod --namespace kube-system |grep intel-fpga-plugin'
-  out 'Check if it runs in 'region' mode:'
-  command "kubectl logs $(kubectl  get pods --namespace kube-system |grep intel-fpga-plugin|cut -f1 -d' ') --namespace kube-system"
+  out '4. Deploy FPGA plugin'
+  command 'kubectl apply -k deployments/fpga_plugin/overlays/region'
+  sleep 2
+  out 'Check if its pods are running:'
+  command 'kubectl get pods -n intelfpgaplugin-system'
+  out 'Deploy the mappings:'
+  command 'kubectl apply -f deployments/fpga_admissionwebhook/mappings-collection.yaml'
+  out 'Check webhook pod logs:'
+  command "kubectl logs $(kubectl get pods -n intelfpgaplugin-system| grep intelfpgaplugin-webhook | awk '{print $1}') -n intelfpgaplugin-system"
+  out 'Check if the plugin runs in 'region' mode:'
+  command "kubectl logs $(kubectl  get pods --namespace intelfpgaplugin-system |grep fpgadeviceplugin|cut -f1 -d' ') --namespace intelfpgaplugin-system"
   out 'Check if resource fpga.intel.com/region-<FPGA interface id> is allocatable:'
   command 'kubectl describe node  |grep -A5 Allocatable'
 }
 
-screen6()
+screen5()
 {
   clear
   cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
-  out '6. Run OPAE workload that uses NLB3 bitstream'
+  out '5. Run OPAE workload that uses NLB3 bitstream'
   out 'Program devices with a NLB0 bitstream that is not wanted by the workload:'
   command "sudo /opt/intel/fpga-sw/fpgatool -b /srv/intel.com/fpga/Arria10.dcp1.2/nlb0.gbs -d ${DEVICE_PREFIX}.0 pr"
   command "sudo /opt/intel/fpga-sw/fpgatool -b /srv/intel.com/fpga/Arria10.dcp1.2/nlb0.gbs -d ${DEVICE_PREFIX}.1 pr"
@@ -154,7 +133,7 @@ screen6()
   command 'cat /sys/class/*/*/*/afu_id'
 }
 
-screen7()
+screen6()
 {
   clear
   out 'Summary:' 15
@@ -171,7 +150,7 @@ if [ "$1" == 'play' ] ; then
   if [ -n "$2" ] ; then
     screen$2
   else
-    for n in $(seq 7) ; do screen$n ; sleep 3; done
+    for n in $(seq 6) ; do screen$n ; sleep 3; done
   fi
 elif [ "$1" == 'cleanup' ] ; then
   cleanup

demo/screencast-fpga-preprogrammed.sh

@@ -39,8 +39,7 @@ cleanup()
   out 'Cleanup demo artifacts' 200
   command 'kubectl delete pod test-fpga-preprogrammed || true' 200
   command 'kubectl delete -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml || true' 200
-  command 'kubectl delete -k plugins/deployments/fpga_admissionwebhook/default || true' 200
-  command './plugins/scripts/deploy-fpgaplugin.sh cleanup' 200
+  command 'kubectl delete namespace intelfpgaplugin-system || true' 20
   command 'kubectl annotate node --all fpga.intel.com/device-plugin-mode-' 200
   command 'rm -rf plugins' 200
 }
@@ -66,7 +65,7 @@ screen1()
   sleep 2
   out 'Check if docker is running k8s pods:'
   command 'docker ps --format "table {{.Names}}"'
-  sleep 2
+  sleep 1
 }
 
 screen2()
@@ -75,43 +74,35 @@ screen2()
   rm -rf plugins
   out '2. Clone Intel Device Plugins for Kubernetes repository'
   command "git clone https://github.com/intel/intel-device-plugins-for-kubernetes plugins" 15
+  sleep 1
 }
 
 screen3()
 {
   clear
-  out '3. Deploy admission controller webhook'
-  command 'kubectl apply -k plugins/deployments/fpga_admissionwebhook/default'
+  out '3. Deploy FPGA plugin'
+  command 'kubectl apply -k plugins/deployments/fpga_plugin/overlays/af'
   sleep 3
   out 'Deploy example mappings:'
   command 'kubectl apply -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml'
   sleep 3
-  out 'Check if webhook pod is running:'
-  command 'kubectl get pods -n intelfpgawebhook-system'
+  out 'Check if the plugin pods are running:'
+  command 'kubectl get pods -n intelfpgaplugin-system'
   sleep 2
-  out 'Check its logs:'
-  command "kubectl logs $(kubectl get pods -n intelfpgawebhook-system| grep intelfpgawebhook-controller-manager | awk '{print $1}') -n intelfpgawebhook-system"
-  sleep 2
-}
-
-screen4()
-{
-  clear
-  out '5. Deploy FPGA plugin'
-  command './plugins/scripts/deploy-fpgaplugin.sh'
-  sleep 4
-  out 'Check if its pod is runnning:'
-  command 'kubectl get pod | grep intel-fpga-plugin'
+  out 'Check webhook pod logs:'
+  command "kubectl logs $(kubectl get pods -n intelfpgaplugin-system| grep intelfpgaplugin-webhook | awk '{print $1}') -n intelfpgaplugin-system"
+  out 'Check if the plugin runs in 'region' mode:'
+  command "kubectl logs $(kubectl  get pods --namespace intelfpgaplugin-system |grep fpgadeviceplugin|cut -f1 -d' ') --namespace intelfpgaplugin-system"
   sleep 2
   out 'Check if resource fpga.intel.com/af-<af id> is allocatable:'
   command 'kubectl describe node  |grep -A4 Allocatable'
   sleep 2
 }
 
-screen5()
+screen4()
 {
   clear
-  out '6. Run OPAE workload that uses NLB3 bitstream'
+  out '4. Run OPAE workload that uses NLB3 bitstream'
   out 'Check if devices are programmed with NLB3:'
   command 'cat /sys/class/*/*/*/afu_id'
   out 'Run workload:'
@@ -120,9 +111,10 @@ screen5()
   sleep 5
   out 'Look at the test output'
   command 'kubectl logs test-fpga-preprogrammed'
+  sleep 2
 }
 
-screen6()
+screen5()
 {
   clear
   out 'Summary:' 15
@@ -138,7 +130,7 @@ if [ "$1" == 'play' ] ; then
   if [ -n "$2" ] ; then
     screen$2
   else
-    for n in $(seq 6) ; do screen$n ; sleep 3; done
+    for n in $(seq 5) ; do screen$n ; sleep 3; done
   fi
 elif [ "$1" == 'cleanup' ] ; then
   cleanup