From f3be57792c2a5c43da28fddd60f338b08d7997f1 Mon Sep 17 00:00:00 2001 From: theburi Date: Tue, 2 Mar 2021 12:22:34 +0000 Subject: [PATCH 1/3] addign sharded clusters --- README.md | 19 ++- .../templates/database-shard.yaml | 89 +++++++++++ .../templates/database.yaml | 17 +- .../ent-operator-database/values-shard.yaml | 151 ++++++++++++++++++ charts/ent-operator-database/values.yaml | 15 +- charts/ent-operator-opsmanager/values.yaml | 4 +- helpers/MongoDB-deploy-shard.sh | 72 +++++++++ helpers/MongoDB-deploy.sh | 4 +- 8 files changed, 362 insertions(+), 9 deletions(-) create mode 100644 charts/ent-operator-database/templates/database-shard.yaml create mode 100644 charts/ent-operator-database/values-shard.yaml create mode 100644 helpers/MongoDB-deploy-shard.sh diff --git a/README.md b/README.md index 28a0213f..6e9a1b06 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,24 @@ -# This Repository is NOT a supported MongoDB product ## MongoDB Helm Charts repositry for Kubernetes This functionality is in alpha and is subject to change. The code is provided as-is with no warranties. Alpha features are not subject to the support SLA of official GA features. +# Quick Start + +```helm repo add mongodb https://github.com/mongodb/helm-charts``` +```helm dependency update``` + +In order to install Ops Manager run this command + +```helm upgrade opsmanager . -n opsmanager --create-namespace -i``` + +In order to install MongoDB DataBase: + +```helm upgrade mongodb . --set opsManager.configMap=opsmanager-configmap --set opsManager.secretRef=opsmanager-org-access-key -n $MONGODB_NAMESPACE --create-namespace -i``` + +Where `opsmanager-configmap` and `opsmanager-org-access-key` contain OpsManager connection properties + +Helper script could be found at ./helpers/MongoDB-deploy.sh It contains an example that automates MongoDB Deployment using mongocli ## Charts @@ -12,6 +27,6 @@ This repository contains sample HELM charts for different MongoDB products | charts | |-------------------------| -| ent-operator | +| ent-operator | | ent-operator-database | | ent-operator-opsmanager | diff --git a/charts/ent-operator-database/templates/database-shard.yaml b/charts/ent-operator-database/templates/database-shard.yaml new file mode 100644 index 00000000..8209c46b --- /dev/null +++ b/charts/ent-operator-database/templates/database-shard.yaml @@ -0,0 +1,89 @@ +{{- if eq .Values.type "ShardedCluster" }} +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: {{ .Values.name }} + namespace: {{ .Release.Namespace }} + annotations: + "meta.helm.sh/release-name": {{ .Release.Name }} + "meta.helm.sh/release-namespace": {{ .Release.Namespace }} + labels: + "helm.sh/chart": {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + "app.kubernetes.io/managed-by": {{ .Release.Service }} + +spec: + type: {{ .Values.type | quote }} + version: {{ .Values.version }} + persistent : true + backup: + enabled: {{ .Values.backup }} + +{{- with .Values.mongos }} + {{- toYaml . | nindent 2 }} +{{- end }} + +{{- with .Values.shardServer }} + {{- toYaml . | nindent 2 }} +{{- end }} + +{{- with .Values.configServer }} + {{- toYaml . | nindent 2 }} +{{- end }} + + opsManager: + configMapRef: +{{- if .Values.opsManager.configMap }} + name: {{ .Values.opsManager.configMap }} +{{- else }} + name: {{ .Values.name }}-configmap +{{- end }} +{{- if .Values.opsManager.secretRef }} + credentials: {{ .Values.opsManager.secretRef }} +{{- else }} + credentials: {{ .Values.name }}-credential +{{- end }} + + security: + authentication: + enabled: true + modes: + {{- range .Values.security.authentication.modes }} + - {{ . | quote }} # Valid authentication modes are "SCRAM' and "X509" + {{- end }} + {{- if .Values.security.tls.enabled }} + tls: + enabled: {{ .Values.security.tls.enabled }} + ca: {{ .Values.security.tls.caRef }} + {{- end }} + + # Optional field - ignoreUnknownUsers + # A value of true means that any users not configured via the Operator or the Ops Manager or Cloud Manager UI + # will not be altered in any way + + # If you need to manage MongoDB users directly via the mongods, set this value to true + ignoreUnknownUsers: false # default value false + podSpec: + podTemplate: + spec: + terminationGracePeriodSeconds: 10 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + foo: bar + # This container will be added to each pod as a sidecar + containers: + - name: mongodb-enterprise-database + resources: + {{- toYaml .Values.resources | nindent 14 }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 10 }} + {{- end }} + + persistence: + {{- toYaml .Values.persistence | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/charts/ent-operator-database/templates/database.yaml b/charts/ent-operator-database/templates/database.yaml index dd100a94..d1ca330a 100644 --- a/charts/ent-operator-database/templates/database.yaml +++ b/charts/ent-operator-database/templates/database.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.type "ReplicaSet" }} --- apiVersion: mongodb.com/v1 kind: MongoDB @@ -13,7 +14,7 @@ metadata: spec: type: {{ .Values.type | quote }} - members: {{ .Values.members }} + members: {{ .Values.replicaSet.members }} # Using a version >= 4.0 will enable SCRAM-SHA-256 authentication # setting a version < 4.0 will enable SCRAM-SHA-1/MONGODB-CR authentication @@ -54,6 +55,17 @@ spec: # If you need to manage MongoDB users directly via the mongods, set this value to true ignoreUnknownUsers: false # default value false +{{- if .Values.externalConnectivity.enable }} + externalConnectivity: + {{- with .Values.externalConnectivity.replicaSetHorizons }} + replicaSetHorizons: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} +{{- if .Values.agent }} + agent: +{{- toYaml .Values.agent | nindent 4 }} +{{- end }} podSpec: podTemplate: spec: @@ -76,4 +88,5 @@ spec: {{- end }} persistence: - {{- toYaml .Values.persistence | nindent 6 }} \ No newline at end of file + {{- toYaml .Values.persistence | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/charts/ent-operator-database/values-shard.yaml b/charts/ent-operator-database/values-shard.yaml new file mode 100644 index 00000000..1e513222 --- /dev/null +++ b/charts/ent-operator-database/values-shard.yaml @@ -0,0 +1,151 @@ +## MongoDB Enterprise Database + +# Set this to true if your cluster is managing SecurityContext for you. +# If running OpenShift (Cloud, Minishift, etc.), set this to true. +managedSecurityContext: false + +# Section to describe components that needs to be installed +mongodb-enterprise-operator: + enabled: true + watchOpsManager: false + watchDatabase: true + +# Optional configuration. +deployValidationWebhooks: true + +name: mdbreplset +type: ShardedCluster +version: 4.4.2-ent +backup: false + +mongos: + mongodsPerShardCount: 3 + mongosCount: 2 + mongosPodSpec: + # additionalMongodConfig: + persistence: + single: 1G + # multiple: + # data: + # journal: + # logs: + podTemplate: + # metadata: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.8" + memory: 1G + # tolerations: + # - key: "key" + # operator: "Exists" + # effect: "NoSchedule" + agentConfigServer: + startupOptions: + maxLogFiles: "30" + dialTimeoutSeconds: "40" + +shardServer: + shardCount: 2 + shardPodSpec: + # additionalMongodConfig: + persistence: + single: 1G + # multiple: + # data: + # journal: + # logs: + podTemplate: + # metadata: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.8" + memory: 1G + # tolerations: + # - key: "key" + # operator: "Exists" + # effect: "NoSchedule" + agentConfigServer: + startupOptions: + maxLogFiles: "30" + dialTimeoutSeconds: "40" + +configServer: + configServerCount: 3 + configSrvPodSpec: + # additionalMongodConfig: + persistence: + single: 1G + # multiple: + # data: + # journal: + # logs: + podTemplate: + # metadata: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.8" + memory: 1G + # tolerations: + # - key: "key" + # operator: "Exists" + # effect: "NoSchedule" + agentConfigServer: + startupOptions: + maxLogFiles: "30" + dialTimeoutSeconds: "40" + +opsManager: + # Ops Manager connection could be configured with Values and This HELM chart will create + # nesessary Secret and Config Map. + URL: + orgid: + APIKey: + APISecret: + # Alternatevly an existing secret and config map could be provided directly + configMap: opsmanager-configmap + secretRef: opsmanager-org-access-key + +security: + authentication: + modes: ["SCRAM"] # Valid authentication modes are "SCRAM", "LDAP" and "X509" + tls: + enabled: false + caRef: mdbreplset-ca + # Note: Operator would expect Pem secret to have name: -cert + +clusterName: cluster.local + +registry: + imagePullSecrets: + # TODO: specify for each image and move there? + pullPolicy: Always + # Specify if images are pulled from private registry + +users: + - username: admin-user + db: admin + password: "%SomeLong%password$foradmin" + roles: + - db: admin + name: clusterAdmin + - db: admin + name: userAdminAnyDatabase + - db: admin + name: readWrite + - db: admin + name: userAdminAnyDatabase + - username: app-user + db: admin + password: "%SomeLong%password$" + roles: + - db: admin + name: readWrite diff --git a/charts/ent-operator-database/values.yaml b/charts/ent-operator-database/values.yaml index ac07f698..0dabb890 100644 --- a/charts/ent-operator-database/values.yaml +++ b/charts/ent-operator-database/values.yaml @@ -16,9 +16,13 @@ deployValidationWebhooks: true name: mdbreplset type: ReplicaSet version: 4.4.2-ent -members: 3 backup: false +# Cluster type selection +replicaSet: + members: 3 + + opsManager: # Ops Manager connection could be configured with Values and This HELM chart will create # nesessary Secret and Config Map. @@ -70,6 +74,15 @@ persistence: clusterName: cluster.local +# External connectivity configuration +# https://docs.mongodb.com/kubernetes-operator/master/tutorial/connect-from-outside-k8s/ +externalConnectivity: + enable: false + replicaSetHorizons: + - "example-website": "web1.example.com:30907" + - "example-website": "web2.example.com:32350" + - "example-website": "web3.example.com:31185" + registry: imagePullSecrets: # TODO: specify for each image and move there? diff --git a/charts/ent-operator-opsmanager/values.yaml b/charts/ent-operator-opsmanager/values.yaml index b642c1ae..96109452 100644 --- a/charts/ent-operator-opsmanager/values.yaml +++ b/charts/ent-operator-opsmanager/values.yaml @@ -87,10 +87,10 @@ backup: resources: limits: cpu: 2 - memory: 8G + memory: 6G requests: cpu: 1 - memory: 6G + memory: 3G tolerations: [] # Set this to true if the operator will require Kubernetes CA diff --git a/helpers/MongoDB-deploy-shard.sh b/helpers/MongoDB-deploy-shard.sh new file mode 100644 index 00000000..e45ab3bb --- /dev/null +++ b/helpers/MongoDB-deploy-shard.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -euxo pipefail + +# Helper script to automate OpsManager Key creating and setting correct ConfigMap and secret to deploy MongoDB +# Set following MongoDB Env variables: + +# MongoDB name and namespace +MONGODB_NAMESPACE=mongodb +MONGODB_NAME=mongoreplset + +# Information about Ops Manager deployment +OPS_MANAGER_NAMESPACE=opsmanager +OPS_MANAGER_HELM_RELEASE_NAME=opsmanager + +# Name of Ops Manager org where MongoDB will be dployed +OPS_MANAGER_ORG_NAME=DemoOrg + +## Few hints to automate OpsManager after installation and deploy first cluster + +# Before moving on make sure OpsManager CR is running +until [ $(kubectl get om -n $OPS_MANAGER_NAMESPACE -o=jsonpath='{.items[0].status.opsManager.phase}') = Running ]; +do +sleep 10s +done; + +# Use mongocli to simplify setting up Ops Manager. https://docs.mongodb.com/mongocli +# This script assumes that access to OpsManager initial Global Admin key secret is not restricted + +kubectl config set-context --current --namespace=$OPS_MANAGER_NAMESPACE + +### Set up cli profile to connect to OpsManager that was provisioned by Operator +mongocli config set private_api_key $(kubectl -n $OPS_MANAGER_NAMESPACE get secrets/$OPS_MANAGER_HELM_RELEASE_NAME-ops-manager-admin-key --template={{.data.publicApiKey}} | base64 -D) + +mongocli config set public_api_key $(kubectl -n $OPS_MANAGER_NAMESPACE get secrets/$OPS_MANAGER_HELM_RELEASE_NAME-ops-manager-admin-key --template={{.data.user}} | base64 -D) + +# This works for EKS Load Balancer for other OM ingresses please use different aproach +mongocli config set ops_manager_url http://$(kubectl -n $OPS_MANAGER_NAMESPACE get svc $OPS_MANAGER_HELM_RELEASE_NAME-ops-manager-svc-ext -o=jsonpath='{.status.loadBalancer.ingress[0].hostname}'):8080/ + +mongocli config set service "ops-manager" + +OPS_MANAGER_ORG_ID="" +### Create new Organization `mongodbTest` +if [[ $(mongocli iam organizations ls --name $OPS_MANAGER_ORG_NAME -o go-template="{{ len .Results }}") > 0 ]] +then +OPS_MANAGER_ORG_ID=$(mongocli iam organizations ls --name $OPS_MANAGER_ORG_NAME -o go-template="{{ (index .Results 0).ID }}" ) +else +OPS_MANAGER_ORG_ID=$(mongocli iam organizations create $OPS_MANAGER_ORG_NAME -o go-template="{{ .ID }}" ) +fi +### create API Key for the org we just created with Role ORG_OWNER + + +# Note: MongoDB CRD requires a secret +# kubectl create secret generic opsmanager-org-access-key --from-literal="user=LFBQEYDP" --from-literal="publicApiKey=9af0ce8b-c88d-4521-a22f-db5fcacf8a9e" +# Note: Command to create Ops Manager API Key looks like: +# mongocli iam organizations apiKeys create --orgId $(mongocli iam organizations list --name mongodbTest -o go-template="{{ range .Results }} {{ .ID }} {{ end }}") --desc "My API key" --role ORG_OWNER -o go-template="{{ .PrivateKey }} {{.PublicKey}}" +kubectl create ns $MONGODB_NAMESPACE || true +kubectl config set-context --current --namespace=$MONGODB_NAMESPACE + +# cleanup of the existing cm and secret +kubectl delete secret opsmanager-org-access-key || true +kubectl delete configmap opsmanager-configmap || true + +# By putting two commands together we will get a somewhat complicated command that creates OpsManage API Key and generates Kubernetes secret `opsmanager-org-access-key` +mongocli iam organizations apiKeys create --orgId $(mongocli iam organizations describe $OPS_MANAGER_ORG_ID -o go-template="{{ .ID }}") --desc "My API key" --role ORG_OWNER -o go-template='kubectl -n '${MONGODB_NAMESPACE}' create secret generic opsmanager-org-access-key --from-literal="user={{.PublicKey}}" --from-literal="publicApiKey={{ .PrivateKey }}" ' | bash +# create config map +kubectl create configmap opsmanager-configmap --from-literal=projectName=$MONGODB_NAME --from-literal=baseUrl="http://$(kubectl -n $OPS_MANAGER_NAMESPACE get svc $OPS_MANAGER_HELM_RELEASE_NAME-ops-manager-svc-ext -o=jsonpath='{.status.loadBalancer.ingress[0].hostname}'):8080" --from-literal=orgId=$OPS_MANAGER_ORG_ID +# We could use this secrete to deploy MongoDB CR. + +# Deploy HELM Chart +pushd ../charts/ent-operator-database/ +helm upgrade mongodb . -f values-shard.yaml --set opsManager.configMap=opsmanager-configmap --set opsManager.secretRef=opsmanager-org-access-key -n $MONGODB_NAMESPACE --create-namespace -i +popd \ No newline at end of file diff --git a/helpers/MongoDB-deploy.sh b/helpers/MongoDB-deploy.sh index 9459c3f3..57ea95fc 100644 --- a/helpers/MongoDB-deploy.sh +++ b/helpers/MongoDB-deploy.sh @@ -18,7 +18,7 @@ OPS_MANAGER_ORG_NAME=DemoOrg ## Few hints to automate OpsManager after installation and deploy first cluster # Before moving on make sure OpsManager CR is running -until [ $(kubectl get om -n $OPS_MANAGER_NAMESPACE -o=jsonpath='{.items[0].status.opsManager.phase}') = Running ]; +until [ $(kubectl get om -n $OPS_MANAGER_NAMESPACE -o=jsonpath='{.items[0].status.opsManager.phase}') == Running ]; do sleep 10s done; @@ -40,7 +40,7 @@ mongocli config set service "ops-manager" OPS_MANAGER_ORG_ID="" ### Create new Organization `mongodbTest` -if [ $(mongocli iam organizations ls --name $OPS_MANAGER_ORG_NAME -o go-template="{{ len .Results }}") > 0 ] +if [[ $(mongocli iam organizations ls --name $OPS_MANAGER_ORG_NAME -o go-template="{{ len .Results }}") > 0 ]] then OPS_MANAGER_ORG_ID=$(mongocli iam organizations ls --name $OPS_MANAGER_ORG_NAME -o go-template="{{ (index .Results 0).ID }}" ) else From e3789dccc9c65ab17cebe803160bd5d87608a15e Mon Sep 17 00:00:00 2001 From: theburi Date: Tue, 2 Mar 2021 14:01:07 +0000 Subject: [PATCH 2/3] adding additional config --- charts/ent-operator-database/templates/database.yaml | 7 +++++++ charts/ent-operator-database/values.yaml | 3 +++ charts/ent-operator/values.yaml | 4 ++-- helpers/MongoDB-deploy.sh | 7 ++++++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/charts/ent-operator-database/templates/database.yaml b/charts/ent-operator-database/templates/database.yaml index d1ca330a..b04b2fe4 100644 --- a/charts/ent-operator-database/templates/database.yaml +++ b/charts/ent-operator-database/templates/database.yaml @@ -55,6 +55,13 @@ spec: # If you need to manage MongoDB users directly via the mongods, set this value to true ignoreUnknownUsers: false # default value false + + + additionalMongodConfig: +{{- with .Values.additionalMongodConfig }} +{{- toYaml . | nindent 4 }} +{{- end }} + {{- if .Values.externalConnectivity.enable }} externalConnectivity: {{- with .Values.externalConnectivity.replicaSetHorizons }} diff --git a/charts/ent-operator-database/values.yaml b/charts/ent-operator-database/values.yaml index 0dabb890..f338a15d 100644 --- a/charts/ent-operator-database/values.yaml +++ b/charts/ent-operator-database/values.yaml @@ -74,6 +74,9 @@ persistence: clusterName: cluster.local +additionalMongodConfig: + storageEngine: wiredTiger + # External connectivity configuration # https://docs.mongodb.com/kubernetes-operator/master/tutorial/connect-from-outside-k8s/ externalConnectivity: diff --git a/charts/ent-operator/values.yaml b/charts/ent-operator/values.yaml index dadd6908..7557084b 100644 --- a/charts/ent-operator/values.yaml +++ b/charts/ent-operator/values.yaml @@ -19,7 +19,7 @@ name: mongodb-enterprise-operator deployment_name: mongodb-enterprise-operator # Version of mongodb-enterprise-operator and mongodb-enterprise-database images -version: 1.9.1 +version: 1.9.2 # The Custom Resources that will be watched by the Operator. # Needs to be changed if only some of the CRDs are installed @@ -43,7 +43,7 @@ registry: operator: Image: quay.io/mongodb/mongodb-enterprise-operator - Tag: 1.9.1 + Tag: 1.9.2 database: Image: quay.io/mongodb/mongodb-enterprise-database diff --git a/helpers/MongoDB-deploy.sh b/helpers/MongoDB-deploy.sh index 57ea95fc..caab894b 100644 --- a/helpers/MongoDB-deploy.sh +++ b/helpers/MongoDB-deploy.sh @@ -69,4 +69,9 @@ kubectl create configmap opsmanager-configmap --from-literal=projectName=$MONGOD # Deploy HELM Chart pushd ../charts/ent-operator-database/ helm upgrade mongodb . --set opsManager.configMap=opsmanager-configmap --set opsManager.secretRef=opsmanager-org-access-key -n $MONGODB_NAMESPACE --create-namespace -i -popd \ No newline at end of file +popd + +until [ $(kubectl get mdb -n $MONGODB_NAMESPACE -o=jsonpath='{.items[0].status.phase}') = Running ]; +do +sleep 10s +done; From 41ec344aea36e9d824d603debb4e3235ec7f840e Mon Sep 17 00:00:00 2001 From: theburi Date: Tue, 2 Mar 2021 14:26:27 +0000 Subject: [PATCH 3/3] Rev versions --- charts/ent-operator-database/Chart.yaml | 2 +- charts/ent-operator-opsmanager/Chart.yaml | 2 +- charts/ent-operator/Chart.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/ent-operator-database/Chart.yaml b/charts/ent-operator-database/Chart.yaml index e06e0945..862a59e0 100644 --- a/charts/ent-operator-database/Chart.yaml +++ b/charts/ent-operator-database/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: mongodb-enterprise-database description: MongoDB Kubernetes Enterprise Operator- MongoDB DataBase charts -version: 0.2.3 +version: 0.2.4 kubeVersion: '>=1.15-0' keywords: - mongodb diff --git a/charts/ent-operator-opsmanager/Chart.yaml b/charts/ent-operator-opsmanager/Chart.yaml index 57250a33..fe004fce 100644 --- a/charts/ent-operator-opsmanager/Chart.yaml +++ b/charts/ent-operator-opsmanager/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: mongodb-enterprise-opsmanager description: MongoDB Kubernetes Enterprise Operator - Ops Manager Chart -version: 0.2.4 +version: 0.2.5 kubeVersion: '>=1.15-0' keywords: - opsManager diff --git a/charts/ent-operator/Chart.yaml b/charts/ent-operator/Chart.yaml index bc09c370..ba9b2768 100644 --- a/charts/ent-operator/Chart.yaml +++ b/charts/ent-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: mongodb-enterprise-operator description: MongoDB Kubernetes Enterprise Operator deployment -version: 0.3.0 +version: 0.3.1 kubeVersion: '>=1.16-0' keywords: - mongodb