From 1d609d16427ca21a3a6bcbebf2cd76861090e6fe Mon Sep 17 00:00:00 2001 From: Gary Vermeulen Date: Thu, 20 Oct 2022 15:08:25 +0100 Subject: [PATCH 1/7] charts/gateway 3.0.2 --- charts/gateway/Chart.yaml | 2 +- charts/gateway/README.md | 40 ++++++++- charts/gateway/ci/ci-values.yaml | 2 + charts/gateway/production-values.yaml | 102 +++++++++++++++++----- charts/gateway/templates/NOTES.txt | 44 +++++++--- charts/gateway/templates/configmap.yaml | 52 +++++++++++ charts/gateway/templates/deployment.yaml | 43 +++++++++ charts/gateway/templates/ingress.yaml | 7 +- charts/gateway/values.yaml | 106 +++++++++++++++++------ 9 files changed, 328 insertions(+), 70 deletions(-) diff --git a/charts/gateway/Chart.yaml b/charts/gateway/Chart.yaml index 4fa85c4c..d3723731 100644 --- a/charts/gateway/Chart.yaml +++ b/charts/gateway/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: "10.1.00" description: This Helm Chart deploys the Layer7 Gateway in Kubernetes. name: gateway -version: 3.0.1 +version: 3.0.2 type: application home: https://github.com/CAAPIM/apim-charts maintainers: diff --git a/charts/gateway/README.md b/charts/gateway/README.md index 62502640..c4432101 100644 --- a/charts/gateway/README.md +++ b/charts/gateway/README.md @@ -4,6 +4,21 @@ This Chart deploys the API Gateway v10.x onward with the following `optional` su ### Important Note The included MySQL subChart is enabled by default to make trying this chart out easier. ***It is not supported or recommended for production.*** Layer7 assumes that you are deploying a Gateway solution to a Kubernetes environment with an external MySQL database. +## 3.0.2 General Updates +To reduce reliance on requiring a custom gateway image for custom and modular assertions, scripts and restman bundles a bootstrap script has been introduced. The script works with the /opt/docker/custom folder. + +The best way to populate this folder is with an initContainer where files can be copied directly across or dynamically loaded from an external source. +- [InitContainer Examples](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples) + +The following configuration options have been added +- [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraints-for-pods) +- [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) +- [Pod Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) +- [Container Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) +- Http headers can also now be added to the liveness and readiness probes +- Ingress API Version validation has been updated to check for available APIs vs. KubeVersion +- SubCharts now show image repository and tags + ### Upgrading to Chart v3.0.0 Please see the 3.0.0 updates, this release brings significant updates and ***breaking changes*** if you are using an external Hazelcast 3.x server. Services and Ingress configuration have also changed. Read the 3.0.0 Updates below and check out the [additional guides](#additional-guides) for more info. @@ -152,12 +167,11 @@ database: * [Java Args](#java-args) * [System Properties](#system-properties) * [Gateway Bundles](#bundle-configuration) +* [Bootstrap Script](#bootstrap-script) * [Logs & Audit Configuration](#logs--audit-configuration) * [Autoscaling](#autoscaling) * [RBAC Parameters](#rbac-parameters) * [Service Metrics Demo](#service-metrics-demo) - - * [SubChart Configuration](#subchart-configuration) ## Configuration @@ -240,6 +254,15 @@ The following table lists the configurable parameters of the Gateway chart and t | `readinessProbe.failureThreshold` | Failure Threshold | `10` | | `resources.limits` | Resource Limits | `{}` | | `resources.requests` | Resource Requests | `{}` | +| `nodeSelector` | [Node Selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) | `{}` | +| `affinity` | [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | `{}` | +| `topologySpreadConstraints` | [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraints-for-pods) | `[]` | +| `tolerations` | [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | +| `podSecurityContext` | [Pod Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) | `[]` | +| `containerSecurityContext` | [Container Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) | `{}` | +| `bootstrap.script.enabled` | Enable the bootstrap script | `false` | +| `bootstrap.script.cleanup` | Cleanup the /opt/docker/custom folder | `false` | + ## Port Configuration There are two types of port configuration available in the Gateway Helm Chart that are configured in the following ways @@ -600,6 +623,19 @@ existingBundle: # secretProviderClass: "secret-provider-class-name" ``` +### Bootstrap Script +To reduce reliance on requiring a custom gateway image for custom and modular assertions, scripts and restman bundles a bootstrap script has been introduced. The script works with the /opt/docker/custom folder. The best way to populate this folder is with an initContainer where files can be copied directly across or dynamically loaded from an external source. + +The following configuration enables the script +``` +bootstrap: + script: + enabled: true + cleanup: false <== set this to true if you'd like to clear the /opt/docker/custom folder after it has run. +``` + +More information on how to use initContainers with examples can be found on the [Layer7 Community Github Utilities Repository](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples). + ### Autoscaling Autoscaling is disabled by default, you will need [metrics server](https://github.com/kubernetes-sigs/metrics-server) in conjunction with the configuration below. In order for Kubernetes to determine when to scale, you will also need to configure resources diff --git a/charts/gateway/ci/ci-values.yaml b/charts/gateway/ci/ci-values.yaml index ad28a9eb..da27fc71 100644 --- a/charts/gateway/ci/ci-values.yaml +++ b/charts/gateway/ci/ci-values.yaml @@ -1,3 +1,5 @@ license: value: SSG_LICENSE accept: true +ingress: + enabled: true diff --git a/charts/gateway/production-values.yaml b/charts/gateway/production-values.yaml index 618edd93..f0d20ed5 100644 --- a/charts/gateway/production-values.yaml +++ b/charts/gateway/production-values.yaml @@ -205,16 +205,16 @@ config: # - Administrative access # - Browser-based administration # - Built-in services - properties: - - name: server - value: A + properties: [] + # - name: server + # value: A tls: enabled: true # privateKey: 00000000000000000000000000000002:ssl clientAuthentication: Optional versions: - #- TLSv1.0 - #- TLSv1.1 + # - TLSv1.0 + # - TLSv1.1 - TLSv1.2 - TLSv1.3 useCipherSuitesOrder: true @@ -266,16 +266,16 @@ config: - Administrative access - Browser-based administration - Built-in services - properties: - - name: server - value: B + properties: [] + # - name: server + # value: B tls: enabled: true # privateKey: 00000000000000000000000000000002:ssl clientAuthentication: Optional versions: - #- TLSv1.0 - #- TLSv1.1 + # - TLSv1.0 + # - TLSv1.1 - TLSv1.2 - TLSv1.3 useCipherSuitesOrder: true @@ -353,6 +353,9 @@ management: # Enable Restman, if DBbacked this setting will persist until manually deleted via Policy Manager. restman: enabled: false + # Enable Graphman (placeholder) + graphman: + enabled: false # This is the username/password used for Policy Manager/Gateway Management. username: admin password: mypassword @@ -446,7 +449,7 @@ ingress: service: port: name: https - #number: + # number: # - host: dev1.ca.com # path: "/" # service: @@ -504,6 +507,7 @@ livenessProbe: # path: /ssg/ping # port: 8443 # scheme: HTTPS +# httpHeaders: [] initialDelaySeconds: 40 timeoutSeconds: 1 periodSeconds: 15 @@ -524,24 +528,52 @@ readinessProbe: successThreshold: 1 failureThreshold: 15 - # nodeSelector: {} # affinity: {} +# ref:https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraints-for-pods +topologySpreadConstraints: [] +# topologySpreadConstraints: +# - maxSkew: 1 +# topologyKey: topology.kubernetes.io/zone +# whenUnsatisfiable: DoNotSchedule +# labelSelector: +# matchLabels: +# app: -gateway + +# ref:https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +# ref:https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: {} + +# ref:https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: {} + +# This script reads files in /opt/docker/custom and moves them into the correct location +# for Gateway startup. Enabling this with an empty /opt/docker/custom folder will have no effect. +################################################################################################# +# We recommend using an initContainer with a shared volume to populate the /opt/docker/custom folder. +# The initContainer can either be built with all of the required files, or dynamically retrieve files +# from an external location. +# See the Readme for details and examples. +bootstrap: + script: + enabled: false + cleanup: false + # Add initContainers to the Gateway -initContainers: {} -## Example: -## initContainers: -## - name: your-image-name -## image: your-image -## imagePullPolicy: Always -## ports: -## - name: portname -## containerPort: 1234 -## +initContainers: [] +# initContainers: +# - name: simple-init +# image: docker.io/layer7api/simple-init:1.0.0 +# imagePullPolicy: Always +# volumeMounts: +# - name: config-directory +# mountPath: /opt/docker/custom ## Add sidecars to the Gateway Deployment. -sidecars: {} +sidecars: [] ## Example: ## sidecars: ## - name: your-image-name @@ -597,7 +629,12 @@ installSolutionKits: # MySQL Bitnami chart - https://github.com/bitnami/charts/tree/master/bitnami/mysql (DO NOT USE IN PRODUCTION!!) mysql: image: - tag: "8.0.22-debian-10-r75" + registry: docker.io + repository: bitnami/mysql + tag: 8.0.22-debian-10-r75 + pullPolicy: IfNotPresent + # pullSecrets: + # - myRegistryKeySecretName auth: username: gateway password: mypassword @@ -651,7 +688,11 @@ hazelcast: external: false # url: hazelcast.example.com:5701 image: + repository: "hazelcast/hazelcast" tag: "5.1.1" + pullPolicy: IfNotPresent + # pullSecrets: + # - myRegistryKeySecretName cluster: memberCount: 2 mancenter: @@ -673,6 +714,12 @@ hazelcast: # This is not a production implementation! influxdb: enabled: false + image: + repository: "influxdb" + tag: "1.8.10-alpine" + pullPolicy: IfNotPresent + # pullSecrets: + # - registry-secret service: port: 8086 persistence: @@ -686,6 +733,13 @@ influxdb: # Settings for Grafana - https://github.com/bitnami/charts/tree/master/bitnami/grafana grafana: enabled: false + image: + registry: docker.io + repository: bitnami/grafana + tag: 9.0.7-debian-11-r0 + pullPolicy: IfNotPresent + # pullSecrets: + # - myRegistryKeySecretName # Change this to update the UI Password admin: user: admin diff --git a/charts/gateway/templates/NOTES.txt b/charts/gateway/templates/NOTES.txt index 6e237098..3a226c4c 100644 --- a/charts/gateway/templates/NOTES.txt +++ b/charts/gateway/templates/NOTES.txt @@ -1,23 +1,39 @@ -Success! +################################################################################## +#### Success! #### +################################################################################## +{{- if .Release.IsInstall}} +#### Your gateway deployment has been INSTALLED #### +{{- else }} +#### Your gateway deployment has been UPGRADED #### +{{- end }} +################################################################################## -{{if .Release.IsInstall}} -Your deployment has been INSTALLED -{{ else }} -Your deployment has been UPGRADED -{{ end }} -To get the Gateway's IP Address type the following -$ kubectl get svc -n {{ .Release.Namespace }} +To view the Gateway's services you can use the following command +$ kubectl get svc -n {{ .Release.Namespace }} | grep {{ .Chart.Name }} -The Gateway is accessible on :8443/9443 +{{- if .Values.ingress.enabled }} + +You configured the following ingress hosts +{{- range .Values.ingress.rules }} +- {{ .host }} +{{- end }} +{{- end }} + +{{- if .Values.grafana.enabled}} -{{if .Values.grafana.enabled}} To access the Grafana Dashboard $ kubectl port-forward svc/{{.Release.Name}}-grafana 3000 -n {{ .Release.Namespace }} Open a browser and navigate to -http://localhost:3000 +- http://localhost:3000 +- username: {{.Values.grafana.admin.user}} +- password: {{.Values.grafana.admin.password}} +{{- end }} + +To learn more about the Gateway Helm Chart check out the following links -username: {{.Values.grafana.admin.user}} -password: {{.Values.grafana.admin.password}} +Gateway Helm Chart Readme +- https://github.com/CAAPIM/apim-charts/tree/stable/charts/gateway -{{ end }} +Thinking in Kubernetes +- https://techdocs.broadcom.com/us/en/ca-enterprise-software/layer7-api-management/api-gateway/congw-10-1/learning-center/thinking-in-kubernetes.html#thinkingk8s diff --git a/charts/gateway/templates/configmap.yaml b/charts/gateway/templates/configmap.yaml index b51df8b5..a8dc5423 100644 --- a/charts/gateway/templates/configmap.yaml +++ b/charts/gateway/templates/configmap.yaml @@ -115,3 +115,55 @@ data: {{- end }} +{{- if .Values.bootstrap.script.enabled }} + 003-parse-custom-files: |- + #!/bin/bash + BASE_CONFIG_DIR="/opt/docker/custom" + BUNDLE_DIR="$BASE_CONFIG_DIR/bundle" + CUSTOM_ASSERTIONS_DIR="$BASE_CONFIG_DIR/custom-assertions" + MODULAR_ASSERTIONS_DIR="$BASE_CONFIG_DIR/modular-assertions" + CUSTOM_SCRIPTS_DIR="$BASE_CONFIG_DIR/scripts/custom" + CUSTOM_HEALTHCHECK_DIR="$BASE_CONFIG_DIR/scripts/healthcheck" + BASE_TARGET_DIR="/opt/SecureSpan/Gateway" + TARGET_BUNDLE_DIR="$BASE_TARGET_DIR/node/default/etc/bootstrap/bundle" + TARGET_CUSTOM_ASSERTIONS_DIR="$BASE_TARGET_DIR/runtime/modules/lib" + TARGET_MODULAR_ASSERTIONS_DIR="$BASE_TARGET_DIR/runtime/modules/assertions" + TARGET_CUSTOM_SCRIPTS_DIR="/opt/docker/rc.d" + TARGET_CUSTOM_HEALTHCHECK_DIR="/opt/docker/rc.d/diagnostic/" + + function cleanup() { + echo "***********************************************" + echo "removing $BASE_CONFIG_DIR" + echo "***********************************************" + rm -rf $BASE_CONFIG_DIR/* + } + + function copy() { + TYPE=$1 + EXT=$2 + SOURCE_DIR=$3 + TARGET_DIR=$4 + echo "***********************************************" + echo "scanning for $TYPE in $SOURCE_DIR" + echo "***********************************************" + FILES=$(find $3 -type f -name '*'$2 2>/dev/null) + for file in $FILES; do + name=$(basename "$file") + if [[ $EXT == ".sh" ]]; then + chmod +x $file + fi + cp $file $4/$name + echo -e "$name written to $4/$name" + done + } + + copy "bundles" ".bundle" $BUNDLE_DIR $TARGET_BUNDLE_DIR + copy "custom assertions" ".jar" $CUSTOM_ASSERTIONS_DIR $TARGET_CUSTOM_ASSERTIONS_DIR + copy "modular assertions" ".aar" $MODULAR_ASSERTIONS_DIR $TARGET_MODULAR_ASSERTIONS_DIR + copy "custom scripts" ".sh" $CUSTOM_SCRIPTS_DIR $TARGET_CUSTOM_SCRIPTS_DIR + copy "custom healthcheck scripts" ".sh" $CUSTOM_HEALTHCHECK_DIR $TARGET_CUSTOM_HEALTHCHECK_DIR +{{- if .Values.bootstrap.cleanup }} + cleanup +{{- end}} + +{{- end }} diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml index 378ce0be..44860cf8 100644 --- a/charts/gateway/templates/deployment.yaml +++ b/charts/gateway/templates/deployment.yaml @@ -24,6 +24,15 @@ spec: {{- if .Values.affinity }} affinity: {{- toYaml .Values.affinity | nindent 12 }} {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- toYaml .Values.tolerations | nindent 12 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- toYaml .Values.topologySpreadConstraints | nindent 12 }} + {{- end }} + {{- if .Values.podSecurityContext }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 12 }} + {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- toYaml .Values.nodeSelector | nindent 12 }} {{- end }} @@ -44,6 +53,9 @@ spec: - name: {{ .Chart.Name }} image: {{.Values.image.registry}}/{{.Values.image.repository}}:{{.Values.image.tag}} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 12 }} + {{- end }} {{- if .Values.installSolutionKits.enabled}} lifecycle: postStart: @@ -118,6 +130,10 @@ spec: - name: {{ template "gateway.fullname" . }}-restman mountPath: /opt/SecureSpan/Gateway/node/default/etc/bootstrap/services/restman {{- end }} +{{- if .Values.management.graphman.enabled }} + - name: {{ template "gateway.fullname" . }}-graphman + mountPath: /opt/SecureSpan/Gateway/node/default/etc/bootstrap/services/graphman +{{- end }} {{- if .Values.existingBundle.enabled }} {{- range .Values.existingBundle.configMaps }} - name: {{ .name }} @@ -139,6 +155,11 @@ spec: mountPath: /opt/docker/rc.d/base/update-service-account-token.xml subPath: update-service-account-token.xml {{- end }} +{{- if .Values.bootstrap.script.enabled }} + - name: {{ template "gateway.fullname" . }}-parse-custom-files-script + mountPath: /opt/docker/rc.d/003-parse-custom-files.sh + subPath: 003-parse-custom-files.sh +{{- end }} {{- if .Values.initContainers }} {{- range .Values.initContainers }} {{- range .volumeMounts }} @@ -177,6 +198,11 @@ spec: path: {{ .Values.livenessProbe.path | default "/ssg/ping"}} port: {{ .Values.livenessProbe.port | default 8443 }} scheme: {{ .Values.livenessProbe.scheme | default "HTTPS" }} + httpHeaders: + {{- range.Values.livenessProbe.httpHeaders }} + - name: {{ .name }} + value: {{ .value }} + {{- end }} {{- else }} exec: command: @@ -196,6 +222,11 @@ spec: path: {{ .Values.readinessProbe.path | default "/ssg/ping"}} port: {{ .Values.readinessProbe.port | default 8443 }} scheme: {{ .Values.readinessProbe.scheme | default "HTTPS" }} + httpHeaders: + {{- range.Values.livenessProbe.httpHeaders }} + - name: {{ .name }} + value: {{ .value }} + {{- end }} {{- else }} exec: command: @@ -273,6 +304,10 @@ spec: - name: {{ template "gateway.fullname" . }}-restman emptyDir: {} {{- end }} +{{- if .Values.management.graphman.enabled }} + - name: {{ template "gateway.fullname" . }}-graphman + emptyDir: {} +{{- end }} {{- if .Values.management.kubernetes.loadServiceAccountToken }} - name: {{ template "gateway.fullname" . }}-service-account-token-script configMap: @@ -287,6 +322,14 @@ spec: - key: update-service-account-token path: update-service-account-token.xml {{- end }} +{{- if .Values.bootstrap.script.enabled }} + - name: {{ template "gateway.fullname" . }}-parse-custom-files-script + configMap: + name: {{ template "gateway.fullname" . }}-configmap + items: + - key: 003-parse-custom-files + path: 003-parse-custom-files.sh +{{- end }} {{- if .Values.existingBundle.enabled }} {{- range .Values.existingBundle.configMaps }} - name: {{ .name }} diff --git a/charts/gateway/templates/ingress.yaml b/charts/gateway/templates/ingress.yaml index 7b5d52d7..3b48f72b 100644 --- a/charts/gateway/templates/ingress.yaml +++ b/charts/gateway/templates/ingress.yaml @@ -1,6 +1,5 @@ {{ if .Values.ingress.enabled }} -{{- $kubeTargetVersion := .Capabilities.KubeVersion.GitVersion }} -{{- if semverCompare ">=1.19-0" $kubeTargetVersion -}} +{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" -}} apiVersion: networking.k8s.io/v1 {{- else -}} apiVersion: networking.k8s.io/v1beta1 @@ -17,7 +16,7 @@ metadata: {{ $key }}: "{{ $val }}" {{- end }} spec: -{{- if semverCompare ">=1.19-0" $kubeTargetVersion }} +{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} ingressClassName: {{ .Values.ingress.ingressClassName }} {{- end }} tls: @@ -28,7 +27,7 @@ spec: - host: {{ .host }} http: paths: -{{- if semverCompare ">=1.19-0" $kubeTargetVersion }} +{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} - pathType: Prefix path: {{ .path }} backend: diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index e94b6885..cc4197ba 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -139,6 +139,8 @@ resources: # memory: 4Gi config: + # Heap Size should be a percentage of the memory configured in resource limits + # by default it is 50% - you should not go above 75% heapSize: "2g" javaArgs: - -Dcom.l7tech.bootstrap.autoTrustSslKey=trustAnchor,TrustedFor.SSL,TrustedFor.SAML_ISSUER @@ -201,9 +203,9 @@ config: # - Administrative access # - Browser-based administration # - Built-in services - properties: - - name: server - value: A + properties: [] + # - name: server + # value: A tls: enabled: true # privateKey: 00000000000000000000000000000002:ssl @@ -262,9 +264,9 @@ config: - Administrative access - Browser-based administration - Built-in services - properties: - - name: server - value: B + properties: [] + # - name: server + # value: B tls: enabled: true # privateKey: 00000000000000000000000000000002:ssl @@ -349,6 +351,9 @@ management: # Enable Restman, if DBbacked this setting will persist until manually deleted via Policy Manager. restman: enabled: false + # Enable Graphman (placeholder) + graphman: + enabled: false # This is the username/password used for Policy Manager/Gateway Management. username: admin password: mypassword @@ -443,12 +448,12 @@ ingress: port: name: https # number: -# - host: dev1.ca.com -# path: "/" -# service: -# port: -# name: anotherport -# #number: + # - host: dev1.ca.com + # path: "/" + # service: + # port: + # name: anotherport + # #number: # Additional Environment variables to be added to the Gateway Configmap additionalEnv: {} @@ -500,6 +505,7 @@ livenessProbe: # path: /ssg/ping # port: 8443 # scheme: HTTPS +# httpHeaders: [] initialDelaySeconds: 40 timeoutSeconds: 1 periodSeconds: 15 @@ -520,24 +526,52 @@ readinessProbe: successThreshold: 1 failureThreshold: 15 - # nodeSelector: {} # affinity: {} +# ref:https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraints-for-pods +topologySpreadConstraints: [] +# topologySpreadConstraints: +# - maxSkew: 1 +# topologyKey: topology.kubernetes.io/zone +# whenUnsatisfiable: DoNotSchedule +# labelSelector: +# matchLabels: +# app: -gateway + +# ref:https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +# ref:https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: {} + +# ref:https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: {} + +# This script reads files in /opt/docker/custom and moves them into the correct location +# for Gateway startup. Enabling this with an empty /opt/docker/custom folder will have no effect. +################################################################################################# +# We recommend using an initContainer with a shared volume to populate the /opt/docker/custom folder. +# The initContainer can either be built with all of the required files, or dynamically retrieve files +# from an external location. +# See the Readme for details and examples. +bootstrap: + script: + enabled: false + cleanup: false + # Add initContainers to the Gateway -initContainers: {} -## Example: -## initContainers: -## - name: your-image-name -## image: your-image -## imagePullPolicy: Always -## ports: -## - name: portname -## containerPort: 1234 -## +initContainers: [] +# initContainers: +# - name: simple-init +# image: docker.io/layer7api/simple-init:1.0.0 +# imagePullPolicy: Always +# volumeMounts: +# - name: config-directory +# mountPath: /opt/docker/custom ## Add sidecars to the Gateway Deployment. -sidecars: {} +sidecars: [] ## Example: ## sidecars: ## - name: your-image-name @@ -593,7 +627,12 @@ installSolutionKits: # MySQL Bitnami chart - https://github.com/bitnami/charts/tree/master/bitnami/mysql (DO NOT USE IN PRODUCTION!!) mysql: image: - tag: "8.0.22-debian-10-r75" + registry: docker.io + repository: bitnami/mysql + tag: 8.0.22-debian-10-r75 + pullPolicy: IfNotPresent + # pullSecrets: + # - myRegistryKeySecretName auth: username: gateway password: mypassword @@ -647,7 +686,11 @@ hazelcast: external: false # url: hazelcast.example.com:5701 image: + repository: "hazelcast/hazelcast" tag: "5.1.1" + pullPolicy: IfNotPresent + # pullSecrets: + # - myRegistryKeySecretName cluster: memberCount: 2 mancenter: @@ -669,6 +712,12 @@ hazelcast: # This is not a production implementation! influxdb: enabled: false + image: + repository: "influxdb" + tag: "1.8.10-alpine" + pullPolicy: IfNotPresent + # pullSecrets: + # - registry-secret service: port: 8086 persistence: @@ -682,6 +731,13 @@ influxdb: # Settings for Grafana - https://github.com/bitnami/charts/tree/master/bitnami/grafana grafana: enabled: false + image: + registry: docker.io + repository: bitnami/grafana + tag: 9.0.7-debian-11-r0 + pullPolicy: IfNotPresent + # pullSecrets: + # - myRegistryKeySecretName # Change this to update the UI Password admin: user: admin From bebc7106c6bbec218656a0c43a9e11ae3e8d48a0 Mon Sep 17 00:00:00 2001 From: Gary Vermeulen Date: Fri, 21 Oct 2022 13:08:10 +0100 Subject: [PATCH 2/7] charts/gateway hpa version check updated, health check target directory updated --- charts/gateway/README.md | 2 +- charts/gateway/templates/configmap.yaml | 2 +- charts/gateway/templates/hpa.yaml | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/charts/gateway/README.md b/charts/gateway/README.md index c4432101..e5f79900 100644 --- a/charts/gateway/README.md +++ b/charts/gateway/README.md @@ -16,7 +16,7 @@ The following configuration options have been added - [Pod Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) - [Container Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) - Http headers can also now be added to the liveness and readiness probes -- Ingress API Version validation has been updated to check for available APIs vs. KubeVersion +- Ingress and HPA API Version validation has been updated to check for available APIs vs. KubeVersion - SubCharts now show image repository and tags ### Upgrading to Chart v3.0.0 diff --git a/charts/gateway/templates/configmap.yaml b/charts/gateway/templates/configmap.yaml index a8dc5423..c1b08b02 100644 --- a/charts/gateway/templates/configmap.yaml +++ b/charts/gateway/templates/configmap.yaml @@ -129,7 +129,7 @@ data: TARGET_CUSTOM_ASSERTIONS_DIR="$BASE_TARGET_DIR/runtime/modules/lib" TARGET_MODULAR_ASSERTIONS_DIR="$BASE_TARGET_DIR/runtime/modules/assertions" TARGET_CUSTOM_SCRIPTS_DIR="/opt/docker/rc.d" - TARGET_CUSTOM_HEALTHCHECK_DIR="/opt/docker/rc.d/diagnostic/" + TARGET_CUSTOM_HEALTHCHECK_DIR="/opt/docker/rc.d/diagnostic/health_check" function cleanup() { echo "***********************************************" diff --git a/charts/gateway/templates/hpa.yaml b/charts/gateway/templates/hpa.yaml index 30d0947c..4e0e3094 100644 --- a/charts/gateway/templates/hpa.yaml +++ b/charts/gateway/templates/hpa.yaml @@ -1,6 +1,5 @@ {{- if .Values.autoscaling.enabled }} -{{- $kubeTargetVersion := .Capabilities.KubeVersion.GitVersion }} -{{- if semverCompare ">=1.23-0" $kubeTargetVersion -}} +{{- if $.Capabilities.APIVersions.Has "autoscaling/v2" -}} apiVersion: autoscaling/v2 {{- else -}} apiVersion: autoscaling/v2beta2 From f022c206ae6f45925fd15620dec6d1e8b75902b8 Mon Sep 17 00:00:00 2001 From: Gary Vermeulen Date: Sat, 22 Oct 2022 20:53:35 +0100 Subject: [PATCH 3/7] charts/gateway updated documentation, added custom healthchecks, added custom configuration options --- charts/gateway/README.md | 68 +++++++++++++++++++- charts/gateway/production-values.yaml | 35 ++++++++++- charts/gateway/templates/configmap.yaml | 47 ++++++++++---- charts/gateway/templates/deployment.yaml | 80 +++++++++++++++++++++++- charts/gateway/values.yaml | 33 ++++++++++ 5 files changed, 244 insertions(+), 19 deletions(-) diff --git a/charts/gateway/README.md b/charts/gateway/README.md index e5f79900..7c165a05 100644 --- a/charts/gateway/README.md +++ b/charts/gateway/README.md @@ -5,12 +5,14 @@ This Chart deploys the API Gateway v10.x onward with the following `optional` su The included MySQL subChart is enabled by default to make trying this chart out easier. ***It is not supported or recommended for production.*** Layer7 assumes that you are deploying a Gateway solution to a Kubernetes environment with an external MySQL database. ## 3.0.2 General Updates -To reduce reliance on requiring a custom gateway image for custom and modular assertions, scripts and restman bundles a bootstrap script has been introduced. The script works with the /opt/docker/custom folder. +To reduce reliance on requiring a custom/derived gateway image for custom and modular assertions, scripts and restman bundles a bootstrap script has been introduced. The script works with the /opt/docker/custom folder. The best way to populate this folder is with an initContainer where files can be copied directly across or dynamically loaded from an external source. -- [InitContainer Examples](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples) +- [InitContainer Examples](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples) - this repository also contains examples for custom health checks and configuration files. The following configuration options have been added +- [Custom Health Checks](#custom-health-checks) +- [Custom Configuration Files](#custom-configuration-files) - [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraints-for-pods) - [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) - [Pod Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) @@ -168,6 +170,8 @@ database: * [System Properties](#system-properties) * [Gateway Bundles](#bundle-configuration) * [Bootstrap Script](#bootstrap-script) +* [Custom Health Checks](#custom-health-checks) +* [Custom Configuration Files](#custom-configuration-files) * [Logs & Audit Configuration](#logs--audit-configuration) * [Autoscaling](#autoscaling) * [RBAC Parameters](#rbac-parameters) @@ -634,8 +638,68 @@ bootstrap: cleanup: false <== set this to true if you'd like to clear the /opt/docker/custom folder after it has run. ``` +The bootstrap script scans files in ```/opt/docker/custom```. This folder is populated by an initContainer. + +The following folder stucture must be maintained + +- Restman Bundles (.bundle) + - Source ```/opt/docker/custom/bundles``` + - Target ```/opt/SecureSpan/Gateway/node/default/etc/bootstrap/bundle``` +- Custom Assertions (.jar) + - Source ```/opt/docker/custom/custom-assertions``` + - Target ```/opt/SecureSpan/Gateway/runtime/modules/lib/``` +- Modular Assertions (.aar) + - Source ```/opt/docker/custom/modular-assertions``` + - Target ```/opt/SecureSpan/Gateway/runtime/modules/assertions``` +- Properties (.properties) + - Source ```/opt/docker/custom/properties``` + - Target ```/opt/SecureSpan/Gateway/node/default/etc/conf/``` + + More information on how to use initContainers with examples can be found on the [Layer7 Community Github Utilities Repository](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples). +### Custom Health Checks +You can now specify a configMap or Secret that contains healthcheck scripts. These are mounted to ```/opt/docker/rc.d/diagnostic/health_check``` where they are run by ```/opt/docker/rc.d/diagnostic/health_check.sh```. + +- Limited to a single configmap or secret. + - ConfigMaps and Secrets can hold multiple scripts. + - [See this example](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples) + +***NOTE: if you set a configMap and a Secret only one of them will be applied to your API Gateway.*** +``` +existingHealthCheck: + enabled: false + configMap: {} + # name: healthcheck-scripts-configmap + # defaultMode: 292 + # optional: false + secret: {} + # name: healthcheck-scripts-secret + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "vault-database" +``` + +### Custom Configuration Files +Certain folders on the Container Gateway are not writeable by design. This configuration allows you to mount existing configMap/Secret keys to specific paths on the Gateway without the need for a root user or a custom/derived image. + +- [See this example](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples) +``` +customConfig: + enabled: false + # mounts: + # - name: sampletrafficloggerca-override + # mountPath: /opt/SecureSpan/Gateway/node/default/etc/conf/sampletrafficloggerca.properties + # subPath: sampletrafficloggerca.properties + # secret: + # name: config-override-secret + # item: + # key: sampletrafficloggerca.properties + # path: sampletrafficloggerca.properties +``` + ### Autoscaling Autoscaling is disabled by default, you will need [metrics server](https://github.com/kubernetes-sigs/metrics-server) in conjunction with the configuration below. In order for Kubernetes to determine when to scale, you will also need to configure resources diff --git a/charts/gateway/production-values.yaml b/charts/gateway/production-values.yaml index f0d20ed5..d8935203 100644 --- a/charts/gateway/production-values.yaml +++ b/charts/gateway/production-values.yaml @@ -138,8 +138,6 @@ resources: cpu: 2000m memory: 4Gi - - config: # Heap Size should be a percentage of the memory configured in resource limits # by default it is 50% - you should not go above 75% @@ -491,6 +489,38 @@ existingBundle: # secretProviderClass: "secret-provider-class-name" # - name: mysecretbundle2 + # This is limited to a single configmap or secret. ConfigMaps and Secrets can hold multiple scripts. +# See an example here - https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples +# NOTE: if you set a configMap and a Secret only one of them will be applied to your API Gateway. +existingHealthCheck: + enabled: false + configMap: {} + # name: healthcheck-scripts-configmap + # defaultMode: 292 + # optional: false + secret: {} + # name: healthcheck-scripts-secret + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "vault-database" + + # Certain folders on the Container Gateway are not writeable by design. +# This configuration allows you to mount configMap/Secret keys to specific paths on the Gateway without +# the need for a root user or a custom/derived image. +customConfig: + enabled: false + # mounts: + # - name: sampletrafficloggerca-override + # mountPath: /opt/SecureSpan/Gateway/node/default/etc/conf/sampletrafficloggerca.properties + # subPath: sampletrafficloggerca.properties + # secret: + # name: config-override-secret + # item: + # key: sampletrafficloggerca.properties + # path: sampletrafficloggerca.properties + # This mounts a bundle folder to the Gateway. This requires you to clone the chart repo and populate the bundle folder. # Note that there is a 1MB limit for Configmaps/Secrets so if your bundles exceed that total, the Chart will fail to install/upgrade. # Helm also keeps a revision of each deployment that will include the bundle definition, if that exceeds 1MB then the same error will occur. @@ -522,6 +552,7 @@ readinessProbe: # path: /ssg/ping # port: 8443 # scheme: HTTPS +# httpHeaders: [] initialDelaySeconds: 40 timeoutSeconds: 1 periodSeconds: 15 diff --git a/charts/gateway/templates/configmap.yaml b/charts/gateway/templates/configmap.yaml index c1b08b02..1c7e16f9 100644 --- a/charts/gateway/templates/configmap.yaml +++ b/charts/gateway/templates/configmap.yaml @@ -115,6 +115,7 @@ data: {{- end }} +{{- if .Values.bootstrap }} {{- if .Values.bootstrap.script.enabled }} 003-parse-custom-files: |- #!/bin/bash @@ -122,19 +123,22 @@ data: BUNDLE_DIR="$BASE_CONFIG_DIR/bundle" CUSTOM_ASSERTIONS_DIR="$BASE_CONFIG_DIR/custom-assertions" MODULAR_ASSERTIONS_DIR="$BASE_CONFIG_DIR/modular-assertions" - CUSTOM_SCRIPTS_DIR="$BASE_CONFIG_DIR/scripts/custom" - CUSTOM_HEALTHCHECK_DIR="$BASE_CONFIG_DIR/scripts/healthcheck" + CUSTOM_SHELL_SCRIPTS_DIR="$BASE_CONFIG_DIR/scripts" BASE_TARGET_DIR="/opt/SecureSpan/Gateway" TARGET_BUNDLE_DIR="$BASE_TARGET_DIR/node/default/etc/bootstrap/bundle" TARGET_CUSTOM_ASSERTIONS_DIR="$BASE_TARGET_DIR/runtime/modules/lib" TARGET_MODULAR_ASSERTIONS_DIR="$BASE_TARGET_DIR/runtime/modules/assertions" - TARGET_CUSTOM_SCRIPTS_DIR="/opt/docker/rc.d" - TARGET_CUSTOM_HEALTHCHECK_DIR="/opt/docker/rc.d/diagnostic/health_check" + + error() { + # Send errors to stderr in case these get handled differently by the container PaaS on which this runs + echo "ERROR - ${1}" 1>&2 + exit 1 + } function cleanup() { - echo "***********************************************" + echo "***************************************************************************" echo "removing $BASE_CONFIG_DIR" - echo "***********************************************" + echo "***************************************************************************" rm -rf $BASE_CONFIG_DIR/* } @@ -143,27 +147,44 @@ data: EXT=$2 SOURCE_DIR=$3 TARGET_DIR=$4 - echo "***********************************************" + echo "***************************************************************************" echo "scanning for $TYPE in $SOURCE_DIR" - echo "***********************************************" + echo "***************************************************************************" FILES=$(find $3 -type f -name '*'$2 2>/dev/null) for file in $FILES; do name=$(basename "$file") - if [[ $EXT == ".sh" ]]; then - chmod +x $file - fi cp $file $4/$name echo -e "$name written to $4/$name" done } + + function run() { + TYPE=$1 + EXT=$2 + SOURCE_DIR=$3 + echo "***************************************************************************" + echo "scanning for $TYPE in $SOURCE_DIR" + echo "***************************************************************************" + FILES=$(find $3 -type f -name '*'$2 2>/dev/null) + for file in $FILES; do + name=$(basename "$file") + chmod +x $file + echo -e "running $name" + /bin/bash $file + if [ $? -ne 0 ]; then + echo "Failed executing the script: $file" + exit 1 + fi + done + } copy "bundles" ".bundle" $BUNDLE_DIR $TARGET_BUNDLE_DIR copy "custom assertions" ".jar" $CUSTOM_ASSERTIONS_DIR $TARGET_CUSTOM_ASSERTIONS_DIR copy "modular assertions" ".aar" $MODULAR_ASSERTIONS_DIR $TARGET_MODULAR_ASSERTIONS_DIR - copy "custom scripts" ".sh" $CUSTOM_SCRIPTS_DIR $TARGET_CUSTOM_SCRIPTS_DIR - copy "custom healthcheck scripts" ".sh" $CUSTOM_HEALTHCHECK_DIR $TARGET_CUSTOM_HEALTHCHECK_DIR + run "custom shell scripts" ".sh" $CUSTOM_SHELL_SCRIPTS_DIR {{- if .Values.bootstrap.cleanup }} cleanup {{- end}} +{{- end}} {{- end }} diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml index 44860cf8..3e7e109c 100644 --- a/charts/gateway/templates/deployment.yaml +++ b/charts/gateway/templates/deployment.yaml @@ -130,10 +130,39 @@ spec: - name: {{ template "gateway.fullname" . }}-restman mountPath: /opt/SecureSpan/Gateway/node/default/etc/bootstrap/services/restman {{- end }} -{{- if .Values.management.graphman.enabled }} +{{- if .Values.management.graphman }} + {{- if .Values.management.graphman.enabled }} - name: {{ template "gateway.fullname" . }}-graphman mountPath: /opt/SecureSpan/Gateway/node/default/etc/bootstrap/services/graphman + {{- end }} +{{- end }} +{{- if .Values.existingHealthCheck }} +{{- if .Values.existingHealthCheck.enabled }} + {{- if .Values.existingHealthCheck.configMap }} + - name: {{ .Values.existingHealthCheck.configMap.name }} + mountPath: /opt/docker/rc.d/diagnostic/health_check + {{- end }} + {{- if .Values.existingHealthCheck.secret }} + - name: {{ .Values.existingHealthCheck.secret.name }} + mountPath: /opt/docker/rc.d/diagnostic/health_check + {{ if .Values.existingHealthCheck.secret.csi }} + readOnly: {{ .Values.existingHealthCheck.secret.csi.readOnly }} + {{ end }} + {{- end }} +{{- end }} +{{- end }} + +{{- if .Values.customConfig }} +{{- if .Values.customConfig.enabled }} + {{- range .Values.customConfig.mounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + {{- end }} +{{- end }} {{- end }} + + {{- if .Values.existingBundle.enabled }} {{- range .Values.existingBundle.configMaps }} - name: {{ .name }} @@ -155,11 +184,14 @@ spec: mountPath: /opt/docker/rc.d/base/update-service-account-token.xml subPath: update-service-account-token.xml {{- end }} + +{{- if .Values.bootstrap }} {{- if .Values.bootstrap.script.enabled }} - name: {{ template "gateway.fullname" . }}-parse-custom-files-script mountPath: /opt/docker/rc.d/003-parse-custom-files.sh subPath: 003-parse-custom-files.sh {{- end }} +{{- end }} {{- if .Values.initContainers }} {{- range .Values.initContainers }} {{- range .volumeMounts }} @@ -264,6 +296,26 @@ spec: - key: hazelcast-xml path: hazelcast-client.xml {{- end }} +{{- if .Values.customConfig }} +{{- if .Values.customConfig.enabled }} + {{- range .Values.customConfig.mounts }} + - name: {{ .name }} + {{- if .configMap }} + configMap: + name: {{ .configMap.name }} + items: + - key: {{ .configMap.item.key }} + path: {{ .configMap.item.path }} + {{- else }} + secret: + secretName: {{ .secret.name }} + items: + - key: {{ .secret.item.key }} + path: {{ .secret.item.path }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} {{- if .Values.config.log.override }} - name: {{ template "gateway.fullname" . }}-log-config-override configMap: @@ -304,9 +356,11 @@ spec: - name: {{ template "gateway.fullname" . }}-restman emptyDir: {} {{- end }} -{{- if .Values.management.graphman.enabled }} +{{- if .Values.management.graphman }} + {{- if .Values.management.graphman.enabled }} - name: {{ template "gateway.fullname" . }}-graphman emptyDir: {} + {{- end }} {{- end }} {{- if .Values.management.kubernetes.loadServiceAccountToken }} - name: {{ template "gateway.fullname" . }}-service-account-token-script @@ -322,6 +376,7 @@ spec: - key: update-service-account-token path: update-service-account-token.xml {{- end }} +{{- if .Values.bootstrap }} {{- if .Values.bootstrap.script.enabled }} - name: {{ template "gateway.fullname" . }}-parse-custom-files-script configMap: @@ -330,6 +385,7 @@ spec: - key: 003-parse-custom-files path: 003-parse-custom-files.sh {{- end }} +{{- end }} {{- if .Values.existingBundle.enabled }} {{- range .Values.existingBundle.configMaps }} - name: {{ .name }} @@ -352,6 +408,26 @@ spec: {{- end }} {{- end }} {{- end }} +{{- if .Values.existingHealthCheck }} +{{- if .Values.existingHealthCheck.enabled }} + {{- if .Values.existingHealthCheck.configMap }} + - name: {{ .Values.existingHealthCheck.configMap.name }} + configMap: + defaultMode: {{ .Values.existingHealthCheck.configMap.defaultMode | default 292 }} + optional: {{ .Values.existingHealthCheck.configMap.optional | default false }} + name: {{ .Values.existingHealthCheck.configMap.name }} + {{- end }} + {{- if .Values.existingHealthCheck.secret }} + - name: {{ .Values.existingHealthCheck.secret.name }} + {{- if .Values.existingHealthCheck.secret.csi }} + csi: {{ toYaml .Values.existingHealthCheck.secret.csi | nindent 12 }} + {{- else }} + secret: + secretName: {{ .Values.existingHealthCheck.secret.name }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} {{- if .Values.initContainers }} {{- range .Values.initContainers }} {{- range .volumeMounts }} diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index cc4197ba..8d86a14c 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -489,6 +489,38 @@ existingBundle: # secretProviderClass: "secret-provider-class-name" # - name: mysecretbundle2 +# This is limited to a single configmap or secret. ConfigMaps and Secrets can hold multiple scripts. +# See an example here - https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples +# NOTE: if you set a configMap and a Secret only one of them will be applied to your API Gateway. +existingHealthCheck: + enabled: false + configMap: {} + # name: healthcheck-scripts-configmap + # defaultMode: 292 + # optional: false + secret: {} + # name: healthcheck-scripts-secret + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "vault-database" + +# Certain folders on the Container Gateway are not writeable by design. +# This configuration allows you to mount configMap/Secret keys to specific paths on the Gateway without +# the need for a root user or a custom/derived image. +customConfig: + enabled: false + # mounts: + # - name: sampletrafficloggerca-override + # mountPath: /opt/SecureSpan/Gateway/node/default/etc/conf/sampletrafficloggerca.properties + # subPath: sampletrafficloggerca.properties + # secret: + # name: config-override-secret + # item: + # key: sampletrafficloggerca.properties + # path: sampletrafficloggerca.properties + # This mounts a bundle folder to the Gateway. This requires you to clone the chart repo and populate the bundle folder. # Note that there is a 1MB limit for Configmaps/Secrets so if your bundles exceed that total, the Chart will fail to install/upgrade. # Helm also keeps a revision of each deployment that will include the bundle definition, if that exceeds 1MB then the same error will occur. @@ -520,6 +552,7 @@ readinessProbe: # path: /ssg/ping # port: 8443 # scheme: HTTPS +# httpHeaders: [] initialDelaySeconds: 40 timeoutSeconds: 1 periodSeconds: 15 From 6a0b565bd0c1f6d526f36cbc61875ede59237c06 Mon Sep 17 00:00:00 2001 From: Gary Vermeulen Date: Sat, 22 Oct 2022 21:23:48 +0100 Subject: [PATCH 4/7] charts/gateway updated readme --- charts/gateway/README.md | 59 ++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/charts/gateway/README.md b/charts/gateway/README.md index 7c165a05..26930569 100644 --- a/charts/gateway/README.md +++ b/charts/gateway/README.md @@ -4,26 +4,6 @@ This Chart deploys the API Gateway v10.x onward with the following `optional` su ### Important Note The included MySQL subChart is enabled by default to make trying this chart out easier. ***It is not supported or recommended for production.*** Layer7 assumes that you are deploying a Gateway solution to a Kubernetes environment with an external MySQL database. -## 3.0.2 General Updates -To reduce reliance on requiring a custom/derived gateway image for custom and modular assertions, scripts and restman bundles a bootstrap script has been introduced. The script works with the /opt/docker/custom folder. - -The best way to populate this folder is with an initContainer where files can be copied directly across or dynamically loaded from an external source. -- [InitContainer Examples](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples) - this repository also contains examples for custom health checks and configuration files. - -The following configuration options have been added -- [Custom Health Checks](#custom-health-checks) -- [Custom Configuration Files](#custom-configuration-files) -- [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraints-for-pods) -- [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) -- [Pod Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) -- [Container Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) -- Http headers can also now be added to the liveness and readiness probes -- Ingress and HPA API Version validation has been updated to check for available APIs vs. KubeVersion -- SubCharts now show image repository and tags - -### Upgrading to Chart v3.0.0 -Please see the 3.0.0 updates, this release brings significant updates and ***breaking changes*** if you are using an external Hazelcast 3.x server. Services and Ingress configuration have also changed. Read the 3.0.0 Updates below and check out the [additional guides](#additional-guides) for more info. - ## Prerequisites - Kubernetes 1.22.x - Helm v3.7.x @@ -45,10 +25,36 @@ Please see the 3.0.0 updates, this release brings significant updates and ***bre * [Additional Guides](#additional-guides) * [Thinking in Kubernetes](https://techdocs.broadcom.com/us/en/ca-enterprise-software/layer7-api-management/api-gateway/congw-10-1/learning-center/thinking-in-kubernetes.html) +#### Getting Started +***If you are using a previous version of this Chart please read the updates section before you upgrade.*** +* [Install the Chart](#installing-the-chart) +* [Upgrade the Chart](#upgrading-the-chart) +* [Uninstall the Chart](#uninstalling-the-chart) + # Java 11 API Gateway is now running with Java 11 with the release of the v10.1.00. The Gateway chart's version has been incremented to 2.0.2. -Things to note and be aware of are the deprecation of TLSv1.0/TLSv1.1 and the JAVA_HOME dir has gone through some changes as well. +Things to note and be aware of are the deprecation of TLSv1.0/TLSv1.1 and the JAVA_HOME dir has gone through some changes as well. + +## 3.0.2 General Updates +To reduce reliance on requiring a custom/derived gateway image for custom and modular assertions, scripts and restman bundles a bootstrap script has been introduced. The script works with the /opt/docker/custom folder. + +The best way to populate this folder is with an initContainer where files can be copied directly across or dynamically loaded from an external source. +- [InitContainer Examples](https://github.com/Layer7-Community/Utilities/tree/main/gateway-init-container-examples) - this repository also contains examples for custom health checks and configuration files. + +The following configuration options have been added +- [Custom Health Checks](#custom-health-checks) +- [Custom Configuration Files](#custom-configuration-files) +- [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraints-for-pods) +- [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) +- [Pod Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) +- [Container Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) +- Http headers can also now be added to the liveness and readiness probes +- Ingress and HPA API Version validation has been updated to check for available APIs vs. KubeVersion +- SubCharts now show image repository and tags + +### Upgrading to Chart v3.0.0 +Please see the 3.0.0 updates, this release brings significant updates and ***breaking changes*** if you are using an external Hazelcast 3.x server. Services and Ingress configuration have also changed. Read the 3.0.0 Updates below and check out the [additional guides](#additional-guides) for more info. ## 3.0.0 Updates to Hazelcast ***Hazelcast 4.x/5.x servers are now supported*** this represents a breaking change if you have configured an external Hazelcast 3.x server. @@ -126,21 +132,20 @@ Inspect and update the new gateway-values.yaml $ helm upgrade my-ssg --set-file "license.value=path/to/license.xml" --set "license.accept=true" -f ./gateway-values.yaml layer7/gateway ``` -# Install the Chart +## Installing the Chart Check out [this guide](https://techdocs.broadcom.com/us/en/ca-enterprise-software/layer7-api-management/api-gateway/congw-10-1/learning-center/thinking-in-kubernetes/hands-on-gateway-deployment-in-kubernetes.html) for more in-depth instruction ``` $ helm repo add layer7 https://caapim.github.io/apim-charts/ $ helm repo update $ helm install my-ssg --set-file "license.value=path/to/license.xml" --set "license.accept=true" layer7/gateway ``` - -## Upgrade this Chart -To upgrade the Gateway deployment +## Upgrading the Chart +To upgrade your Gateway Release ``` $ helm upgrade my-ssg --set-file "license.value=path/to/license.xml" --set "license.accept=true" layer7/gateway ``` -## Remove this Chart -To delete Gateway installation +## Uninstalling the Chart +To uninstall the Gateway Chart ``` $ helm uninstall -n From 6a94e0da1edd42d3ae0305d04c2ce6fac427c325 Mon Sep 17 00:00:00 2001 From: Gary Vermeulen Date: Tue, 8 Nov 2022 10:36:28 +0000 Subject: [PATCH 5/7] updated kubernetes version support, updated appVersion, updated default image --- charts/gateway/Chart.yaml | 2 +- charts/gateway/README.md | 9 ++++++--- charts/gateway/production-values.yaml | 2 +- charts/gateway/values.yaml | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/charts/gateway/Chart.yaml b/charts/gateway/Chart.yaml index d3723731..0f99eabe 100644 --- a/charts/gateway/Chart.yaml +++ b/charts/gateway/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: "10.1.00" +appVersion: "10.1.00_CR2" description: This Helm Chart deploys the Layer7 Gateway in Kubernetes. name: gateway version: 3.0.2 diff --git a/charts/gateway/README.md b/charts/gateway/README.md index 26930569..b07b02a2 100644 --- a/charts/gateway/README.md +++ b/charts/gateway/README.md @@ -5,7 +5,8 @@ This Chart deploys the API Gateway v10.x onward with the following `optional` su The included MySQL subChart is enabled by default to make trying this chart out easier. ***It is not supported or recommended for production.*** Layer7 assumes that you are deploying a Gateway solution to a Kubernetes environment with an external MySQL database. ## Prerequisites -- Kubernetes 1.22.x +- Kubernetes 1.24.x + - [Refer to techdocs](https://techdocs.broadcom.com/us/en/ca-enterprise-software/layer7-api-management/api-gateway/congw-10-1/release-notes_cgw/container-gateway-platform-support.html#concept.dita_3277fc35fde9c5232f0d64d7a360181d5d18fd6c) for the latest version support - Helm v3.7.x - Gateway v10.x License @@ -32,11 +33,13 @@ The included MySQL subChart is enabled by default to make trying this chart out * [Uninstall the Chart](#uninstalling-the-chart) # Java 11 -API Gateway is now running with Java 11 with the release of the v10.1.00. The Gateway chart's version has been incremented to 2.0.2. +The Layer7 API Gateway is now running with Java 11 with the release of the v10.1.00. The Gateway chart's version has been incremented to 2.0.2. Things to note and be aware of are the deprecation of TLSv1.0/TLSv1.1 and the JAVA_HOME dir has gone through some changes as well. ## 3.0.2 General Updates +***The default image tag in values.yaml and production-values.yaml now points at specific CR versions of the API Gateway. The appVersion in Chart.yaml has also be updated to reflect that. As of this release that is 10.1.00_CR2*** + To reduce reliance on requiring a custom/derived gateway image for custom and modular assertions, scripts and restman bundles a bootstrap script has been introduced. The script works with the /opt/docker/custom folder. The best way to populate this folder is with an initContainer where files can be copied directly across or dynamically loaded from an external source. @@ -195,7 +198,7 @@ The following table lists the configurable parameters of the Gateway chart and t | `license.accept` | Accept Gateway license EULA | `false` | | `image.registry` | Image Registry | `docker.io` | | `image.repository` | Image Repository | `caapim/gateway` | -| `image.tag` | Image tag | `10.1.00` | +| `image.tag` | Image tag | `10.1.00_CR2` | | `image.pullPolicy` | Image Pull Policy | `IfNotPresent` | | `imagePullSecret.enabled` | Configures Gateway Deployment to use imagePullSecret, you can also leave this disabled and associate an image pull secret with the Gateway's Service Account | `false` | | `imagePullSecret.existingSecretName` | Point to an existing Image Pull Secret | `commented out` | diff --git a/charts/gateway/production-values.yaml b/charts/gateway/production-values.yaml index d8935203..1d175778 100644 --- a/charts/gateway/production-values.yaml +++ b/charts/gateway/production-values.yaml @@ -12,7 +12,7 @@ license: image: registry: docker.io repository: caapim/gateway - tag: 10.1.00 + tag: 10.1.00_CR2 pullPolicy: IfNotPresent # If you are using a Hazelcast 3.x server then you need to set hazelcast.legacy.enabled=true diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index 8d86a14c..01ab868d 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -12,7 +12,7 @@ license: image: registry: docker.io repository: caapim/gateway - tag: 10.1.00 + tag: 10.1.00_CR2 pullPolicy: IfNotPresent # If you are using a Hazelcast 3.x server then you need to set hazelcast.legacy.enabled=true From 6d098c855a027d9f1dcc03a452dc123adeb4ebfa Mon Sep 17 00:00:00 2001 From: Philippe Brand PAM Date: Fri, 25 Nov 2022 16:04:06 +0100 Subject: [PATCH 6/7] - Add custom label keypairs for deployment object Can be needed on some cloud providers such as Tencent --- charts/gateway/templates/deployment.yaml | 3 +++ charts/gateway/values.yaml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml index 3e7e109c..aa11bb62 100644 --- a/charts/gateway/templates/deployment.yaml +++ b/charts/gateway/templates/deployment.yaml @@ -19,6 +19,9 @@ spec: labels: app: {{ template "gateway.fullname" . }} release: {{ .Release.Name }} + {{- range $key, $val := .Values.customlabels }} + {{ $key }}: "{{ $val }}" + {{- end }} spec: serviceAccountName: {{ include "gateway.serviceAccountName" . }} {{- if .Values.affinity }} diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index 01ab868d..b9a4ec38 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -127,6 +127,11 @@ autoscaling: value: 100 periodSeconds: 15 +# Add custom label keypairs for deployment object +# Can be needed on some cloud providers such as Tencent +#customlabels: +# internet-network: allow + resources: # There are no resource limits set by default, this is a consicious choice for the user and # increases the chance of these running on environments with fewer resources available From 03e72b941e11e0b23754e2d3d7656a55207871fe Mon Sep 17 00:00:00 2001 From: Philippe Brand PAM Date: Fri, 25 Nov 2022 16:06:33 +0100 Subject: [PATCH 7/7] - Changes in both prod and std values.yaml --- charts/gateway/production-values.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/gateway/production-values.yaml b/charts/gateway/production-values.yaml index 1d175778..c320f160 100644 --- a/charts/gateway/production-values.yaml +++ b/charts/gateway/production-values.yaml @@ -127,6 +127,11 @@ autoscaling: value: 100 periodSeconds: 15 +# Add custom label keypairs for deployment object +# Can be needed on some cloud providers such as Tencent +#customlabels: +# internet-network: allow + resources: # There are no resource limits set by default, this is a consicious choice for the user and # increases the chance of these running on environments with fewer resources available