Skip to content

Commit a894379

Browse files
zzzeekzzzeek
committed
add multi-master option to galera
This change adds the new value enableMultiMaster to the Galera CR defaulting to false; when set to true, the "statefulset.kubernetes.io/pod-name" key is removed from the mariadb service spec, and when set to false it is restored. This has the effect of publishing the service either under one specific pod for active/passive use, or using any pod for any request which is multi-master use. the value of the attribute is also propigated out to the status within the MariaDBDatabase CR, which is then consumable by openstack services in a similar manner as the TLS settings. Openstack services will then want to set wsrep_sync_wait accordingly for AP or MM mode, which will be proposed in followup changes. Fixes: OSPRH-7406
1 parent 8c6251d commit a894379

15 files changed

+285
-10
lines changed

‎api/bases/mariadb.openstack.org_galeras.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ spec:
6464
description: When TLS is configured, only allow connections to the
6565
DB over TLS
6666
type: boolean
67+
enableMultiMaster:
68+
default: false
69+
description: Use multi-master service routing
70+
type: boolean
6771
logToDisk:
6872
description: Log Galera pod's output to disk
6973
type: boolean
@@ -104,6 +108,7 @@ spec:
104108
type: object
105109
required:
106110
- containerImage
111+
- enableMultiMaster
107112
- replicas
108113
- secret
109114
- storageClass

‎api/bases/mariadb.openstack.org_mariadbdatabases.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ spec:
9898
- type
9999
type: object
100100
type: array
101+
enableMultiMaster:
102+
description: whether the DB instance is using multi-master routing
103+
type: boolean
101104
hash:
102105
additionalProperties:
103106
type: string

‎api/v1beta1/galera_types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ type GaleraSpecCore struct {
8989
// +kubebuilder:validation:Optional
9090
// Log Galera pod's output to disk
9191
LogToDisk bool `json:"logToDisk"`
92+
// +kubebuilder:default=false
93+
// Use multi-master service routing
94+
EnableMultiMaster bool `json:"enableMultiMaster"`
9295
}
9396

9497
// GaleraAttributes holds startup information for a Galera host

‎api/v1beta1/mariadbdatabase_types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ type MariaDBDatabaseStatus struct {
5555

5656
// Whether TLS is supported by the DB instance
5757
TLSSupport bool `json:"tlsSupport,omitempty"`
58+
59+
// whether the DB instance is using multi-master routing
60+
EnableMultiMaster bool `json:"enableMultiMaster,omitempty"`
5861
}
5962

6063
//+kubebuilder:object:root=true

‎config/crd/bases/mariadb.openstack.org_galeras.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ spec:
6464
description: When TLS is configured, only allow connections to the
6565
DB over TLS
6666
type: boolean
67+
enableMultiMaster:
68+
default: false
69+
description: Use multi-master service routing
70+
type: boolean
6771
logToDisk:
6872
description: Log Galera pod's output to disk
6973
type: boolean
@@ -104,6 +108,7 @@ spec:
104108
type: object
105109
required:
106110
- containerImage
111+
- enableMultiMaster
107112
- replicas
108113
- secret
109114
- storageClass

‎config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ spec:
9898
- type
9999
type: object
100100
type: array
101+
enableMultiMaster:
102+
description: whether the DB instance is using multi-master routing
103+
type: boolean
101104
hash:
102105
additionalProperties:
103106
type: string

‎controllers/galera_controller.go

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -509,16 +509,23 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
509509
// service get stuck.
510510
controllerutil.AddFinalizer(service, helper.GetFinalizer())
511511

512-
// NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes)
513-
// by setting labels in the service's label selectors.
514-
// This label is dynamically set based on the status of the Galera cluster,
515-
// so in this CreateOrPatch block we must reuse whatever is present in
516-
// the existing service CR in case we're patching it.
517-
activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey]
518-
service.Spec = pkgsvc.Spec
519-
if present {
520-
service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod
512+
if !instance.Spec.EnableMultiMaster {
513+
// NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes)
514+
// by setting labels in the service's label selectors.
515+
// This label is dynamically set based on the status of the Galera cluster,
516+
// so in this CreateOrPatch block we must reuse whatever is present in
517+
// the existing service CR in case we're patching it.
518+
activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey]
519+
service.Spec = pkgsvc.Spec
520+
521+
if present {
522+
service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod
523+
}
524+
} else {
525+
service.Spec = pkgsvc.Spec
526+
delete(service.Spec.Selector, mariadb.ActivePodSelectorKey)
521527
}
528+
522529
err := controllerutil.SetControllerReference(instance, service, r.Client.Scheme())
523530
if err != nil {
524531
return err

‎controllers/mariadbdatabase_controller.go

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727
ctrl "sigs.k8s.io/controller-runtime"
2828
"sigs.k8s.io/controller-runtime/pkg/client"
2929
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
30+
"sigs.k8s.io/controller-runtime/pkg/handler"
31+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3032

3133
condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
3234
helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper"
@@ -145,7 +147,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
145147

146148
// here we know that Galera exists so add a finalizer to ourselves and to the db CR. Before this point there is no reason to have a finalizer on ourselves as nothing to cleanup.
147149
if instance.DeletionTimestamp.IsZero() || isNewInstance { // this condition can be removed if you wish as it is always true at this point otherwise we would returned earlier.
148-
if controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) {
150+
if dbGalera.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) {
149151
err := r.Update(ctx, dbGalera)
150152
if err != nil {
151153
return ctrl.Result{}, err
@@ -200,6 +202,9 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
200202
return ctrl.Result{}, err
201203
}
202204

205+
// DB instances setup for multi master
206+
instance.Status.EnableMultiMaster = dbGalera.Spec.EnableMultiMaster
207+
203208
dbCreateHash := instance.Status.Hash[databasev1beta1.DbCreateHash]
204209
dbCreateJob := job.NewJob(
205210
jobDef,
@@ -245,8 +250,43 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
245250

246251
// SetupWithManager -
247252
func (r *MariaDBDatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error {
253+
254+
updateStatusFn := func(ctx context.Context, o client.Object) []reconcile.Request {
255+
log := GetLog(ctx, "MariaDBDatabase")
256+
257+
result := []reconcile.Request{}
258+
259+
mariaDBDatabases := &databasev1beta1.MariaDBDatabaseList{}
260+
261+
listOpts := []client.ListOption{
262+
client.InNamespace(o.GetNamespace()),
263+
}
264+
if err := r.Client.List(ctx, mariaDBDatabases, listOpts...); err != nil {
265+
log.Error(err, "Unable to retrieve MariaDBDatabase CRs %w")
266+
return nil
267+
}
268+
269+
for _, cr := range mariaDBDatabases.Items {
270+
271+
if o.GetName() == cr.GetLabels()["dbName"] {
272+
name := client.ObjectKey{
273+
Namespace: o.GetNamespace(),
274+
Name: cr.Name,
275+
}
276+
log.Info(fmt.Sprintf("Galera %s is used by MariaDBDatabase CR %s", o.GetName(), cr.Name))
277+
result = append(result, reconcile.Request{NamespacedName: name})
278+
}
279+
}
280+
281+
if len(result) > 0 {
282+
return result
283+
}
284+
return nil
285+
}
286+
248287
return ctrl.NewControllerManagedBy(mgr).
249288
For(&databasev1beta1.MariaDBDatabase{}).
289+
Watches(&databasev1beta1.Galera{}, handler.EnqueueRequestsFromMapFunc(updateStatusFn)).
250290
Complete(r)
251291
}
252292

‎tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: mariadb.openstack.org/v1beta1
2+
kind: Galera
3+
metadata:
4+
name: openstack
5+
spec:
6+
secret: osp-secret
7+
storageClass: local-storage
8+
storageRequest: 500M
9+
replicas: 3
10+
enableMultiMaster: false
11+
---
12+
apiVersion: mariadb.openstack.org/v1beta1
13+
kind: MariaDBDatabase
14+
metadata:
15+
name: mydatabase
16+
labels:
17+
dbName: openstack
18+
spec:
19+
name: mydatabase

‎tests/kuttl/tests/galera_multi_master/02-assert.yaml

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#
2+
# Check for:
3+
#
4+
# - 1 MariaDB CR
5+
# - 1 Pod for MariaDB CR
6+
#
7+
8+
apiVersion: mariadb.openstack.org/v1beta1
9+
kind: Galera
10+
metadata:
11+
name: openstack
12+
spec:
13+
replicas: 3
14+
secret: osp-secret
15+
storageRequest: 500M
16+
status:
17+
bootstrapped: true
18+
conditions:
19+
- message: Setup complete
20+
reason: Ready
21+
status: "True"
22+
type: Ready
23+
- message: Deployment completed
24+
reason: Ready
25+
status: "True"
26+
type: DeploymentReady
27+
- message: Exposing service completed
28+
reason: Ready
29+
status: "True"
30+
type: ExposeServiceReady
31+
- message: Input data complete
32+
reason: Ready
33+
status: "True"
34+
type: InputReady
35+
- message: RoleBinding created
36+
reason: Ready
37+
status: "True"
38+
type: RoleBindingReady
39+
- message: Role created
40+
reason: Ready
41+
status: "True"
42+
type: RoleReady
43+
- message: ServiceAccount created
44+
reason: Ready
45+
status: "True"
46+
type: ServiceAccountReady
47+
- message: Service config create completed
48+
reason: Ready
49+
status: "True"
50+
type: ServiceConfigReady
51+
- message: Input data complete
52+
reason: Ready
53+
status: "True"
54+
type: TLSInputReady
55+
---
56+
apiVersion: apps/v1
57+
kind: StatefulSet
58+
metadata:
59+
name: openstack-galera
60+
spec:
61+
replicas: 3
62+
selector:
63+
matchLabels:
64+
app: galera
65+
cr: galera-openstack
66+
galera/name: openstack
67+
serviceName: openstack-galera
68+
template:
69+
metadata:
70+
labels:
71+
app: galera
72+
cr: galera-openstack
73+
galera/name: openstack
74+
spec:
75+
containers:
76+
- command:
77+
- /usr/bin/dumb-init
78+
- --
79+
- /usr/local/bin/kolla_start
80+
name: galera
81+
ports:
82+
- containerPort: 3306
83+
name: mysql
84+
protocol: TCP
85+
- containerPort: 4567
86+
name: galera
87+
protocol: TCP
88+
serviceAccount: galera-openstack
89+
serviceAccountName: galera-openstack
90+
status:
91+
availableReplicas: 3
92+
readyReplicas: 3
93+
replicas: 3
94+
---
95+
apiVersion: v1
96+
kind: Pod
97+
metadata:
98+
name: openstack-galera-0
99+
---
100+
apiVersion: v1
101+
kind: Pod
102+
metadata:
103+
name: openstack-galera-1
104+
---
105+
apiVersion: v1
106+
kind: Pod
107+
metadata:
108+
name: openstack-galera-2
109+
---
110+
apiVersion: v1
111+
kind: Service
112+
metadata:
113+
name: openstack-galera
114+
spec:
115+
ports:
116+
- name: mysql
117+
port: 3306
118+
protocol: TCP
119+
targetPort: 3306
120+
selector:
121+
app: galera
122+
cr: galera-openstack
123+
---
124+
apiVersion: mariadb.openstack.org/v1beta1
125+
kind: MariaDBDatabase
126+
metadata:
127+
labels:
128+
dbName: openstack
129+
name: mydatabase
130+
spec:
131+
name: mydatabase

‎tests/kuttl/tests/galera_multi_master/03-assert.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
commands:
5+
- script: |
6+
# ensure kubernetes pod key present
7+
oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-"

‎tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: mariadb.openstack.org/v1beta1
2+
kind: Galera
3+
metadata:
4+
name: openstack
5+
spec:
6+
secret: osp-secret
7+
storageClass: local-storage
8+
storageRequest: 500M
9+
replicas: 3
10+
enableMultiMaster: true

‎tests/kuttl/tests/galera_multi_master/05-assert.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestAssert
3+
commands:
4+
- script: |
5+
# ensure kubernetes pod key *not* present
6+
! oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-"

‎tests/kuttl/tests/galera_multi_master/06-assert.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
apiVersion: mariadb.openstack.org/v1beta1
2+
kind: Galera
3+
metadata:
4+
name: openstack
5+
spec:
6+
replicas: 3
7+
secret: osp-secret
8+
storageRequest: 500M
9+
enableMultiMaster: true
10+
---
11+
apiVersion: mariadb.openstack.org/v1beta1
12+
kind: MariaDBDatabase
13+
metadata:
14+
labels:
15+
dbName: openstack
16+
name: mydatabase
17+
spec:
18+
name: mydatabase
19+
status:
20+
enableMultiMaster: true

‎tests/kuttl/tests/galera_multi_master/07-teardown.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestStep
3+
delete:
4+
- apiVersion: mariadb.openstack.org/v1beta1
5+
kind: Galera
6+
name: openstack
7+
- apiVersion: mariadb.openstack.org/v1beta1
8+
kind: MariaDBDatabase
9+
name: mydatabase
10+
commands:
11+
- script: |
12+
oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2
13+
for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done

0 commit comments

Comments
 (0)