Add Support to store Openwhisk Component Images in Hosted Docker Registry. (apache#447)

Author: neerajmangal

docs/private-docker-registry.md

@@ -0,0 +1,78 @@
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+# Using private docker registry
+
+By default, helm charts currently use docker hub to download images to deploy OpenWhisk components on Kubernetes. If your Kubernetes provider does not allow public docker registry, you can use your hosted docker image registry to deploy OpenWhisk on Kubernetes.
+
+- All openwhisk images should be uploaded to your hosted docker registry server.
+  - openwhisk/apigateway
+  - apache/couchdb
+  - openwhisk/controller
+  - openwhisk/invoker
+  - wurstmeister/kafka
+  - openwhisk/ow-utils
+  - zookeeper
+  - nginx
+  - redis
+  - busybox
+  - openwhisk/alarmprovider
+  - openwhisk/kafkaprovider
+  - openwhisk/cloudantprovider
+
+- Add details of your docker registry information in mycluster.yml.
+
+  ```yaml
+  docker:
+    registry:
+      name: "registry-name/"
+      username: username
+      password: "Passowrd"
+  ```
+
+  > - enabling registry information will cause all your images to be pulled from private docker registry only.
+  > - Append / in your docker registry name.
+
+Enabling *registry.name* will create a docker-registry secret as *{ReleaseName}-private-registry.auth* in Kubernetes which will be used in pod/jobs as *imagePullSecrets*.
+
+```yaml
+# If ReleaseName is owdev and namespace is openwhisk
+# kubectl get secrets owdev-private-registry.auth -o yaml
+
+apiVersion: v1
+data:
+  .dockerconfigjson: <Base64 encoded>
+kind: Secret
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"v1","data":{".dockerconfigjson":"Base64 Encoded"},"kind":"Secret","metadata":{"annotations":{},"labels":{"app":"owdev-openwhisk","chart":"openwhisk-0.1.4","heritage":"Tiller","release":"owdev"},"name":"owdev-private-registry.auth","namespace":"openwhisk"},"type":"kubernetes.io/dockerconfigjson"}
+  creationTimestamp: "2019-04-04T06:44:43Z"
+  labels:
+    app: owdev-openwhisk
+    chart: openwhisk-0.1.4
+    heritage: Tiller
+    release: owdev
+  name: owdev-private-registry.auth
+  namespace: openwhisk
+  resourceVersion: "18273580"
+  selfLink: /api/v1/namespaces/openwhisk/secrets/owdev-private-registry.auth
+  uid: 20f03275-56a5-11e9-9164-005056a3e755
+type: kubernetes.io/dockerconfigjson
+```

helm/openwhisk/templates/_helpers.tpl

@@ -195,3 +195,18 @@ app: {{ template "openwhisk.fullname" . }}
 {{- define "openwhisk.tls_secret_name" -}}
 {{ .Values.whisk.ingress.tls.secretname | default "ow-ingress-tls-secret" | quote }}
 {{- end -}}
+
+{{/* Create imagePullSecrets for private docker-registry*/}}
+{{- define "openwhisk.dockerRegistrySecret" -}}
+{{- if ne .Values.docker.registry.name "" }}
+{{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.docker.registry.name (printf "%s:%s" .Values.docker.registry.username .Values.docker.registry.password | b64enc) | b64enc }}
+{{- end }}
+{{- end -}}
+
+{{/* ImagePullSecrets in pods and job*/}}
+{{- define "openwhisk.docker.imagePullSecrets" -}}
+{{- if ne .Values.docker.registry.name "" }}
+imagePullSecrets:
+- name: {{ .Release.Name }}-private-registry.auth
+{{- end }}
+{{- end -}}

helm/openwhisk/templates/_invoker-helpers.tpl

@@ -33,8 +33,8 @@
 
 {{- define "openwhisk.docker_pull_runtimes" -}}
 - name: docker-pull-runtimes
+  image: "{{- .Values.docker.registry.name -}}{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
   imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }}
-  image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
   command: ["/usr/local/bin/ansible-playbook", "/invoker-scripts/playbook.yml"]
   volumeMounts:
   - name: dockersock

helm/openwhisk/templates/_readiness.tpl

@@ -7,7 +7,7 @@
 # if not db.wipeAndInit, the external db must already be ready; so no need for init container
 {{- else -}}
 - name: "wait-for-couchdb"
-  image: "busybox"
+  image: "{{- .Values.docker.registry.name -}}busybox"
   imagePullPolicy: "IfNotPresent"
   env:
   - name: "READINESS_URL"
@@ -19,7 +19,7 @@
 {{/* Init container that waits for kafka to be ready */}}
 {{- define "openwhisk.readiness.waitForKafka" -}}
 - name: "wait-for-kafka"
-  image: "busybox"
+  image: "{{- .Values.docker.registry.name -}}busybox"
   imagePullPolicy: "IfNotPresent"
   # TODO: I haven't found an easy external test to determine that kafka is up, so as a hack we wait for zookeeper and then sleep for 10 seconds and cross our fingers!
   command: ["sh", "-c", 'result=1; until [ $result -eq 0 ]; do OK=$(echo ruok | nc -w 1 {{ include "openwhisk.zookeeper_zero_host" . }} {{ .Values.zookeeper.port }}); if [ "$OK" == "imok" ]; then result=0; echo "zookeeper returned imok!"; else echo waiting for zookeeper to be ready; sleep 1; fi done; echo "Zookeeper is up; will wait for 10 seconds to give kafka time to initialize"; sleep 10;']
@@ -28,15 +28,15 @@
 {{/* Init container that waits for zookeeper to be ready */}}
 {{- define "openwhisk.readiness.waitForZookeeper" -}}
 - name: "wait-for-zookeeper"
-  image: "busybox"
+  image: "{{- .Values.docker.registry.name -}}busybox"
   imagePullPolicy: "IfNotPresent"
   command: ["sh", "-c", 'result=1; until [ $result -eq 0 ]; do OK=$(echo ruok | nc -w 1 {{ include "openwhisk.zookeeper_zero_host" . }} {{ .Values.zookeeper.port }}); if [ "$OK" == "imok" ]; then result=0; echo "zookeeper returned imok!"; else echo waiting for zookeeper to be ready; sleep 1; fi; done; echo "Success: zookeeper is up"']
 {{- end -}}
 
 {{/* Init container that waits for controller to be ready */}}
 {{- define "openwhisk.readiness.waitForController" -}}
 - name: "wait-for-controller"
-  image: "busybox"
+  image: "{{- .Values.docker.registry.name -}}busybox"
   imagePullPolicy: "IfNotPresent"
   env:
   - name: "READINESS_URL"
@@ -47,7 +47,7 @@
 {{/* Init container that waits for at least 1 healthy invoker */}}
 {{- define "openwhisk.readiness.waitForHealthyInvoker" -}}
 - name: "wait-for-healthy-invoker"
-  image: "busybox"
+  image: "{{- .Values.docker.registry.name -}}busybox"
   imagePullPolicy: "IfNotPresent"
   env:
   - name: "READINESS_URL"

helm/openwhisk/templates/apigateway-pod.yaml

@@ -26,11 +26,11 @@ spec:
 {{ include "openwhisk.affinity.core" . | indent 8 }}
 {{ include "openwhisk.affinity.selfAntiAffinity" ( printf "%s-apigateway" .Release.Name ) | indent 8 }}
       {{- end }}
-
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
         - name: apigateway
           imagePullPolicy: {{ .Values.apigw.imagePullPolicy | quote }}
-          image: "{{- .Values.apigw.imageName -}}:{{- .Values.apigw.imageTag -}}"
+          image: "{{- .Values.docker.registry.name -}}{{- .Values.apigw.imageName -}}:{{- .Values.apigw.imageTag -}}"
           ports:
           - name: mgmt
             containerPort: {{ .Values.apigw.mgmtPort }}

helm/openwhisk/templates/controller-pod.yaml

@@ -34,11 +34,11 @@ spec:
       # The controller must wait for kafka and couchdb to be ready before it starts
 {{ include "openwhisk.readiness.waitForKafka" . | indent 6 }}
 {{ include "openwhisk.readiness.waitForCouchDB" . | indent 6 }}
-
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: controller
         imagePullPolicy: {{ .Values.controller.imagePullPolicy | quote }}
-        image: "{{- .Values.controller.imageName -}}:{{- .Values.controller.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.controller.imageName -}}:{{- .Values.controller.imageTag -}}"
         command: ["/bin/bash", "-c", "/init.sh `hostname | cut -d'-' -f3`"]
         ports:
         - name: controller

helm/openwhisk/templates/couchdb-init-job.yaml

@@ -26,9 +26,10 @@ spec:
       - name: whisk-auth
         secret:
           secretName: {{ .Release.Name }}-whisk.auth
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: init-couchdb
-        image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
         imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }}
         command: ["/bin/bash", "-c", "set -e; . /task/initdb.sh"]
         volumeMounts:

helm/openwhisk/templates/couchdb-pod.yaml

@@ -27,10 +27,10 @@ spec:
 {{ include "openwhisk.affinity.core" . | indent 8 }}
 {{ include "openwhisk.affinity.selfAntiAffinity" ( printf "%s-couchdb" .Release.Name ) | indent 8 }}
       {{- end }}
-
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: couchdb
-        image: "{{- .Values.db.imageName -}}:{{- .Values.db.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.db.imageName -}}:{{- .Values.db.imageTag -}}"
         imagePullPolicy: {{ .Values.db.imagePullPolicy | quote }}
         ports:
         - name: couchdb

helm/openwhisk/templates/install-packages-job.yaml

@@ -24,9 +24,10 @@ spec:
           name: {{ .Release.Name }}-install-packages-cm
       initContainers:
 {{ include "openwhisk.readiness.waitForHealthyInvoker" . | indent 6 }}
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: install-packages
-        image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
         imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }}
         command: ["/bin/bash", "-c", "set -e; . /task/myTask.sh"]
         volumeMounts:

helm/openwhisk/templates/invoker-agent-pod.yaml

@@ -36,10 +36,10 @@ spec:
       initContainers:
       # Pull images for all default runtimes before starting invoker
 {{ include "openwhisk.docker_pull_runtimes" . | indent 6 }}
-
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: invoker-agent
-        image: "{{- .Values.invoker.containerFactory.kubernetes.agent.imageName -}}:{{- .Values.invoker.containerFactory.kubernetes.agent.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.invoker.containerFactory.kubernetes.agent.imageName -}}:{{- .Values.invoker.containerFactory.kubernetes.agent.imageTag -}}"
         imagePullPolicy: {{ .Values.invoker.containerFactory.kubernetes.agent.imagePullPolicy | quote }}
         securityContext:
           privileged: true

helm/openwhisk/templates/invoker-pod.yaml

@@ -53,10 +53,10 @@ spec:
 {{- end }}
       # Wait for a controller to be up (which implies kafka, zookeeper, couchdb are all up as well).
 {{ include "openwhisk.readiness.waitForController" . | indent 6 }}
-
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: invoker
-        image: "{{- .Values.invoker.imageName -}}:{{- .Values.invoker.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.invoker.imageName -}}:{{- .Values.invoker.imageTag -}}"
         imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
 {{- if and (eq .Values.invoker.containerFactory.impl "docker") .Values.invoker.containerFactory.networkConfig.dns.inheritInvokerConfig }}
         command: [ "/bin/bash", "-c", ". /invoker-scripts/configureDNS.sh && /init.sh --uniqueName $INVOKER_NAME" ]

helm/openwhisk/templates/kafka-pod.yaml

@@ -38,10 +38,10 @@ spec:
 
       initContainers:
 {{ include "openwhisk.readiness.waitForZookeeper" . | indent 6 }}
-
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: kafka
-        image: "{{- .Values.kafka.imageName -}}:{{- .Values.kafka.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.kafka.imageName -}}:{{- .Values.kafka.imageTag -}}"
         imagePullPolicy: {{ .Values.kafka.imagePullPolicy | quote }}
 {{- if .Values.k8s.persistence.enabled }}
         volumeMounts:

helm/openwhisk/templates/nginx-pod.yaml

@@ -36,10 +36,10 @@ spec:
           name: {{ .Release.Name }}-nginx
       - name: logs
         emptyDir: {}
-
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: nginx
-        image: "{{- .Values.nginx.imageName -}}:{{- .Values.nginx.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.nginx.imageName -}}:{{- .Values.nginx.imageTag -}}"
         imagePullPolicy: {{ .Values.nginx.imagePullPolicy | quote }}
         ports:
         - name: http

helm/openwhisk/templates/private-registry-secret.yml

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements; and to You under the Apache License, Version 2.0.
+
+{{- if ne .Values.docker.registry.name "" }}
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-private-registry.auth
+  labels:
+{{ include "openwhisk.label_boilerplate" . | indent 4 }}
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: {{ template "openwhisk.dockerRegistrySecret" . }}
+{{- end }}
+

helm/openwhisk/templates/provider-alarm-pod.yaml

@@ -32,9 +32,10 @@ spec:
       initContainers:
       # Wait for a controller to be up (which implies couchdb is up as well).
 {{ include "openwhisk.readiness.waitForController" . | indent 6 }}
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: alarmprovider
-        image: "{{- .Values.providers.alarm.imageName -}}:{{- .Values.providers.alarm.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.providers.alarm.imageName -}}:{{- .Values.providers.alarm.imageTag -}}"
         imagePullPolicy: {{ .Values.providers.alarm.imagePullPolicy | quote }}
         ports:
         - name: alarmprovider

helm/openwhisk/templates/provider-cloudant-pod.yaml

@@ -32,9 +32,10 @@ spec:
       initContainers:
       # Wait for a controller to be up (which implies couchdb is up as well).
 {{ include "openwhisk.readiness.waitForController" . | indent 6 }}
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: cloudantprovider
-        image: "{{- .Values.providers.cloudant.imageName -}}:{{- .Values.providers.cloudant.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.providers.cloudant.imageName -}}:{{- .Values.providers.cloudant.imageTag -}}"
         imagePullPolicy: {{ .Values.providers.cloudant.imagePullPolicy | quote }}
         ports:
         - name: cludntprovider

helm/openwhisk/templates/provider-kafka-pod.yaml

@@ -24,9 +24,10 @@ spec:
       initContainers:
       # Wait for a controller to be up (which implies couchdb and kafka are up as well).
 {{ include "openwhisk.readiness.waitForController" . | indent 6 }}
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
       - name: kafkaprovider
-        image: "{{- .Values.providers.kafka.imageName -}}:{{- .Values.providers.kafka.imageTag -}}"
+        image: "{{- .Values.docker.registry.name -}}{{- .Values.providers.kafka.imageName -}}:{{- .Values.providers.kafka.imageTag -}}"
         imagePullPolicy: {{ .Values.providers.kafka.imagePullPolicy | quote }}
         ports:
         - name: kafkaprovider

helm/openwhisk/templates/redis-pod.yaml

@@ -37,7 +37,7 @@ spec:
 {{- if .Values.k8s.persistence.enabled }}
       initContainers:
       - name: redis-init
-        image: busybox
+        image: "{{- .Values.docker.registry.name -}}busybox"
         command:
           - chown
           - -v
@@ -49,10 +49,10 @@ spec:
           name: redis-data
           readOnly: false
 {{- end }}
-
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }}
       containers:
         - name: redis
-          image: "{{- .Values.redis.imageName -}}:{{- .Values.redis.imageTag -}}"
+          image: "{{- .Values.docker.registry.name -}}{{- .Values.redis.imageName -}}:{{- .Values.redis.imageTag -}}"
           imagePullPolicy: {{ .Values.redis.imagePullPolicy | quote }}
 {{- if .Values.k8s.persistence.enabled }}
           volumeMounts:

helm/openwhisk/templates/tests/package-checker-pod.yaml

@@ -17,9 +17,10 @@ spec:
   - name: task-dir
     configMap:
       name: {{ .Release.Name }}-tests-package-checker
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 2 }}
   containers:
   - name: package-checker
-    image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
+    image: "{{- .Values.docker.registry.name -}}{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
     imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }}
     command: ["/bin/bash", "/task/myTask.sh"]
     volumeMounts:

helm/openwhisk/templates/tests/smoketest-pod.yaml

@@ -17,9 +17,10 @@ spec:
   - name: task-dir
     configMap:
       name: {{ .Release.Name }}-tests-smoketest
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 2 }}
   containers:
   - name: smoketest
-    image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
+    image: "{{- .Values.docker.registry.name -}}{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
     imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }}
     command: ["/bin/bash", "/task/myTask.sh"]
     volumeMounts:

helm/openwhisk/templates/tests/systemtest-pod.yaml

@@ -18,9 +18,10 @@ spec:
   - name: task-dir
     configMap:
       name: {{ .Release.Name }}-tests-systemtest
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 2 }}
   containers:
   - name: systemtest
-    image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
+    image: "{{- .Values.docker.registry.name -}}{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
     imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }}
     command: ["/bin/bash", "/task/myTask.sh"]
     volumeMounts:

helm/openwhisk/templates/wskadmin-pod.yaml

@@ -10,9 +10,10 @@ metadata:
 {{ include "openwhisk.label_boilerplate" . | indent 4 }}
 spec:
   restartPolicy: Always
+{{ include "openwhisk.docker.imagePullSecrets" . | indent 2 }}
   containers:
   - name: wskadmin
-    image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
+    image: "{{- .Values.docker.registry.name -}}{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}"
     imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }}
     command: ["/bin/bash", "-c", "tail -f /dev/null"]
     env: