From 68e9f09e50494f76515917dcad087e0d14373f6f Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Tue, 20 Jun 2023 12:25:32 +0000 Subject: [PATCH] Support TLS for galera Ability to specify a certificate and a CA to be used for galera cluster communication (GCOMM, SST). When the Galera CR is configured to use TLS, the mariadbdatabase CR creates DB users with grants that only allow connecting to the DB over TLS. --- api/bases/mariadb.openstack.org_galeras.yaml | 12 ++ api/v1beta1/galera_types.go | 11 + api/v1beta1/zz_generated.deepcopy.go | 16 ++ .../bases/mariadb.openstack.org_galeras.yaml | 12 ++ config/samples/mariadb_v1beta1_galera.yaml | 2 +- .../mariadb_v1beta1_galera_custom_config.yaml | 2 +- .../samples/mariadb_v1beta1_galera_tls.yaml | 13 ++ controllers/galera_controller.go | 131 +++++++++++- controllers/mariadbdatabase_controller.go | 5 +- pkg/database.go | 15 +- pkg/statefulset.go | 111 +--------- pkg/volumes.go | 197 ++++++++++++++++++ templates/database.sh | 2 +- templates/galera/bin/mysql_bootstrap.sh | 2 +- templates/galera/config/config.json | 28 +++ templates/galera/config/galera_tls.cnf.in | 13 ++ .../galera/config/galera_tls_noca.cnf.in | 12 ++ 17 files changed, 458 insertions(+), 126 deletions(-) create mode 100644 config/samples/mariadb_v1beta1_galera_tls.yaml create mode 100644 templates/galera/config/galera_tls.cnf.in create mode 100644 templates/galera/config/galera_tls_noca.cnf.in diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index 564c69d7..7bced6e8 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -84,6 +84,18 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + tls: + description: TLS settings to use for MySQL and Galera replication + properties: + caSecretName: + description: Secret in the same namespace containing the CA cert + (ca.crt) for client certificate validation + type: string + secretName: + description: Secret in the same namespace containing the server + private key (tls.key) and public cert (tls.crt) for TLS + type: string + type: object required: - containerImage - replicas diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index d1fd93a6..5152266a 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -55,6 +55,17 @@ type GaleraSpec struct { // +kubebuilder:validation:Optional // Adoption configuration AdoptionRedirect AdoptionRedirectSpec `json:"adoptionRedirect"` + // +kubebuilder:validation:Optional + // TLS settings to use for MySQL and Galera replication + TLS TLSSpec `json:"tls,omitempty"` +} + +// TLSSpec defines the TLS options +type TLSSpec struct { + // Secret in the same namespace containing the server private key (tls.key) and public cert (tls.crt) for TLS + SecretName string `json:"secretName,omitempty"` + // Secret in the same namespace containing the CA cert (ca.crt) for client certificate validation + CaSecretName string `json:"caSecretName,omitempty"` } // GaleraAttributes holds startup information for a Galera host diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 145b7314..727ae42c 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -126,6 +126,7 @@ func (in *GaleraSpec) DeepCopyInto(out *GaleraSpec) { } } out.AdoptionRedirect = in.AdoptionRedirect + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GaleraSpec. @@ -374,3 +375,18 @@ func (in *MariaDBStatus) DeepCopy() *MariaDBStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSSpec) DeepCopyInto(out *TLSSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSSpec. +func (in *TLSSpec) DeepCopy() *TLSSpec { + if in == nil { + return nil + } + out := new(TLSSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index 564c69d7..7bced6e8 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -84,6 +84,18 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + tls: + description: TLS settings to use for MySQL and Galera replication + properties: + caSecretName: + description: Secret in the same namespace containing the CA cert + (ca.crt) for client certificate validation + type: string + secretName: + description: Secret in the same namespace containing the server + private key (tls.key) and public cert (tls.crt) for TLS + type: string + type: object required: - containerImage - replicas diff --git a/config/samples/mariadb_v1beta1_galera.yaml b/config/samples/mariadb_v1beta1_galera.yaml index ad73307d..65a0ee4b 100644 --- a/config/samples/mariadb_v1beta1_galera.yaml +++ b/config/samples/mariadb_v1beta1_galera.yaml @@ -3,7 +3,7 @@ kind: Galera metadata: name: openstack spec: - secret: mariadb-secret + secret: osp-secret storageClass: local-storage storageRequest: 500M containerImage: quay.io/podified-antelope-centos9/openstack-mariadb:current-podified diff --git a/config/samples/mariadb_v1beta1_galera_custom_config.yaml b/config/samples/mariadb_v1beta1_galera_custom_config.yaml index 44afffc5..690de0a9 100644 --- a/config/samples/mariadb_v1beta1_galera_custom_config.yaml +++ b/config/samples/mariadb_v1beta1_galera_custom_config.yaml @@ -3,7 +3,7 @@ kind: Galera metadata: name: openstack spec: - secret: mariadb-secret + secret: osp-secret storageClass: local-storage storageRequest: 500M containerImage: quay.io/podified-antelope-centos9/openstack-mariadb:current-podified diff --git a/config/samples/mariadb_v1beta1_galera_tls.yaml b/config/samples/mariadb_v1beta1_galera_tls.yaml new file mode 100644 index 00000000..9892ca30 --- /dev/null +++ b/config/samples/mariadb_v1beta1_galera_tls.yaml @@ -0,0 +1,13 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + containerImage: quay.io/podified-antelope-centos9/openstack-mariadb:current-podified + replicas: 3 + tls: + secretName: galera-tls + caSecretName: galera-tls diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go index ae6b7506..b957a2a4 100644 --- a/controllers/galera_controller.go +++ b/controllers/galera_controller.go @@ -22,12 +22,14 @@ import ( env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + k8s_labels "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/kubectl/pkg/util/podutils" @@ -44,15 +46,21 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" mariadb "github.com/openstack-k8s-operators/mariadb-operator/pkg" ) +const mariaDBReconcileLabel = "mariadb-ref" + // GaleraReconciler reconciles a Galera object type GaleraReconciler struct { client.Client @@ -91,7 +99,7 @@ func buildGcommURI(instance *mariadbv1.Galera) string { res := []string{} for i := 0; i < replicas; i++ { - res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename) + res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename+"."+instance.Namespace+".svc") } uri := "gcomm://" + strings.Join(res, ",") return uri @@ -250,15 +258,17 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal if !found { continue } - ci := instance.Status.Attributes[pod.Name].ContainerID - pci := pod.Status.ContainerStatuses[0].ContainerID - if ci != pci { + // A node can have various attributes depending on its known state. + // A ContainerID attribute is only present if the node is being started. + attrCID := instance.Status.Attributes[pod.Name].ContainerID + podCID := pod.Status.ContainerStatuses[0].ContainerID + if attrCID != "" && attrCID != podCID { // This gcomm URI was pushed in a pod which was restarted // before the attribute got cleared, which means the pod // failed to start galera. Clear the attribute here, and // reprobe the pod's state in the next reconcile loop clearPodAttributes(instance, pod.Name) - util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name) + util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name, "current pod ID", podCID, "recorded ID", attrCID) } } } @@ -454,7 +464,7 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res r.Log.Info(fmt.Sprintf("%s %s database service %s - operation: %s", instance.Kind, instance.Name, service.Name, string(op))) } - // Generate the config maps for the various services + // Generate the config maps configMapVars := make(map[string]env.Setter) err = r.generateConfigMaps(ctx, helper, instance, &configMapVars) if err != nil { @@ -466,16 +476,62 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res err.Error())) return ctrl.Result{}, fmt.Errorf("error calculating configmap hash: %w", err) } + + // + // Extend the config maps with hashes for TLS secrets if any + // + var tlsres ctrl.Result + var tlserror error + if instance.Spec.TLS.SecretName != "" { + tlsres, tlserror = r.checkAndHashTLSSecret(ctx, helper, instance, instance.Spec.TLS.SecretName, &configMapVars) + } + if tlserror == nil && instance.Spec.TLS.CaSecretName != "" { + tlsres, tlserror = r.checkAndHashTLSSecret(ctx, helper, instance, instance.Spec.TLS.CaSecretName, &configMapVars) + } + + if tlserror != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + tlserror.Error())) + + if tlsres.RequeueAfter > 0 { + // TLS secret might be recreated, requeue as this is not fatal + util.LogForObject(helper, "Requeuing until TLS configuration is available", instance, "error", tlserror.Error()) + return tlsres, nil + } + + return tlsres, fmt.Errorf("error calculating configmap hash: %w", tlserror) + } + // From hereon, configMapVars holds a hash of the config generated for this instance + // as well as a hash and of the current TLS certificate and CA used if any. // This is used in an envvar in the statefulset to restart it on config change - envHash := &corev1.EnvVar{} - configMapVars[configMapNameForConfig(instance)](envHash) - instance.Status.ConfigHash = envHash.Value + + keys := make([]string, 0, len(configMapVars)) + for k := range configMapVars { + keys = append(keys, k) + } + sort.Strings(keys) + + hash := "" + for _, k := range keys { + envVar := &corev1.EnvVar{} + configMapVars[k](envVar) + hash = hash + envVar.Value + } + // TODO maybe hash the resulting string to shorten it + instance.Status.ConfigHash = hash instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance), 5) sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper) if sferr != nil { + if k8s_errors.IsNotFound(sferr) { + return ctrl.Result{RequeueAfter: time.Second * 10}, nil + } return sfres, sferr } statefulset := commonstatefulset.GetStatefulSet() @@ -585,7 +641,7 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // a chance to react to all pod's transitions. if statefulset.Status.AvailableReplicas != statefulset.Status.Replicas { util.LogForObject(helper, "Requeuing until all replicas are available", instance) - return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil + return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil } return ctrl.Result{}, err @@ -644,6 +700,44 @@ func (r *GaleraReconciler) generateConfigMaps( return nil } +func (r *GaleraReconciler) checkAndHashTLSSecret( + ctx context.Context, + h *helper.Helper, + instance *mariadbv1.Galera, + secretName string, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + tlsSecret, tlsHash, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: time.Second * 10}, nil + } + return ctrl.Result{}, fmt.Errorf("secret %s not found", secretName) + } + + if value, ok := tlsSecret.Labels[mariaDBReconcileLabel]; !ok || value != instance.Name { + tlsSecret.GetObjectMeta().SetLabels( + k8s_labels.Merge( + tlsSecret.GetObjectMeta().GetLabels(), + map[string]string{ + mariaDBReconcileLabel: instance.Name, + }, + ), + ) + err = r.Client.Update(ctx, tlsSecret) + if err != nil { + if k8s_errors.IsConflict(err) || k8s_errors.IsNotFound(err) { + return ctrl.Result{Requeue: true}, err + } + return ctrl.Result{}, err + } + } + + (*envVars)[secretName] = env.SetValue(tlsHash) + + return ctrl.Result{}, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { r.config = mgr.GetConfig() @@ -653,5 +747,22 @@ func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Service{}). Owns(&corev1.Endpoints{}). Owns(&corev1.ConfigMap{}). + Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc( + func(o client.Object) []reconcile.Request { + labels := o.GetLabels() + + reconcileCR, hasLabel := labels[mariaDBReconcileLabel] + if !hasLabel { + return []reconcile.Request{} + } + + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: reconcileCR, + Namespace: o.GetNamespace(), + }}, + } + }, + )). Complete(r) } diff --git a/controllers/mariadbdatabase_controller.go b/controllers/mariadbdatabase_controller.go index 5dd47e3a..5d6d61bd 100644 --- a/controllers/mariadbdatabase_controller.go +++ b/controllers/mariadbdatabase_controller.go @@ -135,6 +135,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Non-deletion (normal) flow follows // var dbName, dbSecret, dbContainerImage, serviceAccount string + var useTLS bool // It is impossible to reach here without either dbGalera or dbMariadb not being nil, due to the checks above if dbGalera != nil { @@ -147,6 +148,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ dbSecret = dbGalera.Spec.Secret dbContainerImage = dbGalera.Spec.ContainerImage serviceAccount = dbGalera.RbacResourceName() + useTLS = (dbGalera.Spec.TLS.SecretName != "") } else if dbMariadb != nil { if dbMariadb.Status.DbInitHash == "" { r.Log.Info("DB initialization not complete. Requeue...") @@ -157,10 +159,11 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ dbSecret = dbMariadb.Spec.Secret dbContainerImage = dbMariadb.Spec.ContainerImage serviceAccount = dbMariadb.RbacResourceName() + useTLS = false } // Define a new Job object (hostname, password, containerImage) - jobDef, err := mariadb.DbDatabaseJob(instance, dbName, dbSecret, dbContainerImage, serviceAccount) + jobDef, err := mariadb.DbDatabaseJobExt(instance, dbName, dbSecret, dbContainerImage, serviceAccount, useTLS) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/database.go b/pkg/database.go index fdbadc17..f1134b38 100644 --- a/pkg/database.go +++ b/pkg/database.go @@ -14,12 +14,23 @@ type dbCreateOptions struct { DatabaseName string DatabaseHostname string DatabaseAdminUsername string + DatabaseUserTLS string } // DbDatabaseJob - func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string) (*batchv1.Job, error) { + return DbDatabaseJobExt(database, databaseHostName, databaseSecret, containerImage, serviceAccountName, false) +} - opts := dbCreateOptions{database.Spec.Name, databaseHostName, "root"} +// DbDatabaseJobExt - +func DbDatabaseJobExt(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, useTLS bool) (*batchv1.Job, error) { + var tlsStatement string + if useTLS { + tlsStatement = " REQUIRE SSL" + } else { + tlsStatement = "" + } + opts := dbCreateOptions{database.Spec.Name, databaseHostName, "root", tlsStatement} dbCmd, err := util.ExecuteTemplateFile("database.sh", &opts) if err != nil { return nil, err @@ -83,7 +94,7 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s // DeleteDbDatabaseJob - func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string) (*batchv1.Job, error) { - opts := dbCreateOptions{database.Spec.Name, databaseHostName, "root"} + opts := dbCreateOptions{database.Spec.Name, databaseHostName, "root", ""} delCmd, err := util.ExecuteTemplateFile("delete_database.sh", &opts) if err != nil { return nil, err diff --git a/pkg/statefulset.go b/pkg/statefulset.go index 06b1bc1b..47174cfb 100644 --- a/pkg/statefulset.go +++ b/pkg/statefulset.go @@ -106,29 +106,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { ContainerPort: 4567, Name: "galera", }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/secrets", - ReadOnly: true, - Name: "secrets", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraVolumeMounts(g), StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ @@ -153,92 +131,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - Volumes: []corev1.Volume{ - { - Name: "secrets", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: g.Spec.Secret, - Items: []corev1.KeyToPath{ - { - Key: "DbRootPassword", - Path: "dbpassword", - }, - }, - }, - }, - }, - { - Name: "kolla-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "pod-config-data", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "config-data", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "galera.cnf.in", - Path: "galera.cnf.in", - }, - { - Key: mariadbv1.CustomServiceConfigFile, - Path: mariadbv1.CustomServiceConfigFile, - }, - }, - }, - }, - }, - { - Name: "operator-scripts", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-scripts", - }, - Items: []corev1.KeyToPath{ - { - Key: "mysql_bootstrap.sh", - Path: "mysql_bootstrap.sh", - }, - { - Key: "mysql_probe.sh", - Path: "mysql_probe.sh", - }, - { - Key: "detect_last_commit.sh", - Path: "detect_last_commit.sh", - }, - { - Key: "detect_gcomm_and_start.sh", - Path: "detect_gcomm_and_start.sh", - }, - }, - }, - }, - }, - }, + Volumes: getGaleraVolumes(g), }, }, VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ diff --git a/pkg/volumes.go b/pkg/volumes.go index 459131f3..f08e63d5 100644 --- a/pkg/volumes.go +++ b/pkg/volumes.go @@ -1,6 +1,7 @@ package mariadb import ( + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" ) @@ -113,3 +114,199 @@ func getInitVolumeMounts() []corev1.VolumeMount { } } + +func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { + configTemplates := []corev1.KeyToPath{ + { + Key: "galera.cnf.in", + Path: "galera.cnf.in", + }, + { + Key: mariadbv1.CustomServiceConfigFile, + Path: mariadbv1.CustomServiceConfigFile, + }, + } + if g.Spec.TLS.SecretName != "" && g.Spec.TLS.CaSecretName != "" { + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_tls.cnf.in", + Path: "galera_tls.cnf.in", + }) + } else if g.Spec.TLS.SecretName != "" { + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_tls_noca.cnf.in", + Path: "galera_tls_noca.cnf.in", + }) + } + + volumes := []corev1.Volume{ + { + Name: "secrets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: g.Spec.Secret, + Items: []corev1.KeyToPath{ + { + Key: "DbRootPassword", + Path: "dbpassword", + }, + }, + }, + }, + }, + { + Name: "kolla-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: []corev1.KeyToPath{ + { + Key: "config.json", + Path: "config.json", + }, + }, + }, + }, + }, + { + Name: "pod-config-data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: configTemplates, + }, + }, + }, + { + Name: "operator-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-scripts", + }, + Items: []corev1.KeyToPath{ + { + Key: "mysql_bootstrap.sh", + Path: "mysql_bootstrap.sh", + }, + { + Key: "mysql_probe.sh", + Path: "mysql_probe.sh", + }, + { + Key: "detect_last_commit.sh", + Path: "detect_last_commit.sh", + }, + { + Key: "detect_gcomm_and_start.sh", + Path: "detect_gcomm_and_start.sh", + }, + }, + }, + }, + }, + } + + if g.Spec.TLS.SecretName != "" { + volumes = append(volumes, []corev1.Volume{ + { + Name: "tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: g.Spec.TLS.SecretName, + Items: []corev1.KeyToPath{ + { + Key: "tls.key", + Path: "mysql.key", + }, + { + Key: "tls.crt", + Path: "mysql.crt", + }, + }, + }, + }, + }, + }...) + } + + if g.Spec.TLS.CaSecretName != "" { + volumes = append(volumes, []corev1.Volume{ + { + Name: "tls-ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: g.Spec.TLS.CaSecretName, + Items: []corev1.KeyToPath{ + { + Key: "ca.crt", + Path: "ca.crt", + }, + }, + }, + }, + }, + }...) + } + + return volumes +} + +func getGaleraVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data", + ReadOnly: true, + Name: "config-data", + }, { + MountPath: "/var/lib/pod-config-data", + Name: "pod-config-data", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { + MountPath: "/var/lib/kolla/config_files", + ReadOnly: true, + Name: "kolla-config", + }, + } + + if g.Spec.TLS.SecretName != "" { + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ + { + MountPath: "/var/lib/tls", + ReadOnly: true, + Name: "tls", + }, + }...) + } + + if g.Spec.TLS.CaSecretName != "" { + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ + { + MountPath: "/var/lib/tls-ca", + ReadOnly: true, + Name: "tls-ca", + }, + }...) + } + + return volumeMounts +} diff --git a/templates/database.sh b/templates/database.sh index 13782839..95dd6bc2 100755 --- a/templates/database.sh +++ b/templates/database.sh @@ -1,4 +1,4 @@ #!/bin/bash export DatabasePassword=${DatabasePassword:?"Please specify a DatabasePassword variable."} -mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'localhost' IDENTIFIED BY '$DatabasePassword';GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'%' IDENTIFIED BY '$DatabasePassword';" +mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'localhost' IDENTIFIED BY '$DatabasePassword'{{.DatabaseUserTLS}};GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'%' IDENTIFIED BY '$DatabasePassword'{{.DatabaseUserTLS}};" diff --git a/templates/galera/bin/mysql_bootstrap.sh b/templates/galera/bin/mysql_bootstrap.sh index 522ee1fb..544c62c3 100755 --- a/templates/galera/bin/mysql_bootstrap.sh +++ b/templates/galera/bin/mysql_bootstrap.sh @@ -18,7 +18,7 @@ fi # Generate the mariadb configs from the templates, these will get # copied by `kolla_start` when the pod's main container will start -PODNAME=$(hostname -f | cut -d. -f1,2) +PODNAME=$(hostname -f | cut -d. -f1-4) PODIP=$(grep "${PODNAME}" /etc/hosts | cut -d$'\t' -f1) cd /var/lib/config-data for cfg in *.cnf.in; do diff --git a/templates/galera/config/config.json b/templates/galera/config/config.json index 185e33fa..49edcc5e 100644 --- a/templates/galera/config/config.json +++ b/templates/galera/config/config.json @@ -7,6 +7,13 @@ "owner": "root", "perm": "0644" }, + { + "source": "/var/lib/pod-config-data/galera_tls.cnf", + "dest": "/etc/my.cnf.d/galera_tls.cnf", + "owner": "root", + "perm": "0644", + "optional": true + }, { "source": "/var/lib/pod-config-data/galera_custom.cnf", "dest": "/etc/my.cnf.d/galera_custom.cnf", @@ -20,6 +27,27 @@ "owner": "root", "perm": "0755", "merge": "true" + }, + { + "source": "/var/lib/tls/mysql.key", + "dest": "/etc/pki/tls/private/mysql.key", + "owner": "mysql", + "perm": "0755", + "optional": true + }, + { + "source": "/var/lib/tls/mysql.crt", + "dest": "/etc/pki/tls/certs/mysql.crt", + "owner": "mysql", + "perm": "0755", + "optional": true + }, + { + "source": "/var/lib/tls-ca/ca.crt", + "dest": "/etc/ipa/ca.crt", + "owner": "mysql", + "perm": "0755", + "optional": true } ], "permissions": [ diff --git a/templates/galera/config/galera_tls.cnf.in b/templates/galera/config/galera_tls.cnf.in new file mode 100644 index 00000000..2833bb79 --- /dev/null +++ b/templates/galera/config/galera_tls.cnf.in @@ -0,0 +1,13 @@ +[mysqld] +ssl +ssl-cert = /etc/pki/tls/certs/mysql.crt +ssl-key = /etc/pki/tls/private/mysql.key +ssl-cipher = !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +wsrep_provider_options = gcache.recover=no;gmcast.listen_addr=tcp://{ PODIP }:4567;socket.ssl_key=/etc/pki/tls/private/mysql.key;socket.ssl_cert=/etc/pki/tls/certs/mysql.crt;socket.ssl_cipher=AES128-SHA256;socket.ssl_ca=/etc/ipa/ca.crt; +wsrep_sst_method = rsync_tunnel + +[sst] +sockopt = cipher=!SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +tcert = /etc/pki/tls/certs/mysql.crt +tkey = /etc/pki/tls/private/mysql.key +tca = /etc/ipa/ca.crt diff --git a/templates/galera/config/galera_tls_noca.cnf.in b/templates/galera/config/galera_tls_noca.cnf.in new file mode 100644 index 00000000..9c69817e --- /dev/null +++ b/templates/galera/config/galera_tls_noca.cnf.in @@ -0,0 +1,12 @@ +[mysqld] +ssl +ssl-cert = /etc/pki/tls/certs/mysql.crt +ssl-key = /etc/pki/tls/private/mysql.key +ssl-cipher = !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +wsrep_provider_options = gcache.recover=no;gmcast.listen_addr=tcp://{ PODIP }:4567;socket.ssl_key=/etc/pki/tls/private/mysql.key;socket.ssl_cert=/etc/pki/tls/certs/mysql.crt;socket.ssl_cipher=AES128-SHA256; +wsrep_sst_method = rsync_tunnel + +[sst] +sockopt = cipher=!SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +tcert = /etc/pki/tls/certs/mysql.crt +tkey = /etc/pki/tls/private/mysql.key