From 62522b2189e6fcd0cd567415112185b7f223360a Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Tue, 20 Jun 2023 12:25:32 +0000 Subject: [PATCH] Support TLS for galera Ability to specify a certificate and a CA to be used for galera cluster communication (GCOMM, SST). Updates to the certificate used for galera automatically triggers a rolling restart of the galera pods, without service disruption. When the Galera CR is configured to use TLS, the mariadbdatabase CR creates DB users that still allow connection to the DB without using TLS. This is because Openstack clients currently cannot be configured to connect via TLS or via plain TCP. This specific part will be addressed in a subsequent commit. --- api/bases/mariadb.openstack.org_galeras.yaml | 33 +++- api/go.mod | 14 +- api/go.sum | 29 ++- api/v1beta1/conditions.go | 3 + api/v1beta1/galera_types.go | 9 +- api/v1beta1/mariadbdatabase_funcs.go | 3 +- api/v1beta1/zz_generated.deepcopy.go | 13 ++ .../bases/mariadb.openstack.org_galeras.yaml | 33 +++- config/rbac/role.yaml | 2 + config/samples/cert-manager-galera-cert.yaml | 75 ++++++++ .../samples/mariadb_v1beta1_galera_tls.yaml | 14 ++ controllers/galera_controller.go | 182 ++++++++++++++++-- controllers/mariadbdatabase_controller.go | 9 +- go.mod | 4 +- go.sum | 8 +- pkg/mariadb/database.go | 11 +- pkg/mariadb/statefulset.go | 144 ++------------ pkg/mariadb/volumes.go | 179 +++++++++++++++++ templates/database.sh | 2 +- templates/galera/config/config.json | 35 ++++ .../galera/config/galera_external_tls.cnf.in | 8 + templates/galera/config/galera_tls.cnf.in | 15 ++ .../common/assert_sample_deployment.yaml | 4 + .../01-assert.yaml | 44 +++++ .../01-deploy-galera.yaml | 45 +++++ .../01-tls-certificate.yaml | 1 + .../02-assert.yaml | 6 + .../03-teardown.yaml | 19 ++ .../galera_deploy/03-cleanup_galera.yaml | 1 + .../galera_deploy_external_tls/01-assert.yaml | 54 ++++++ .../01-deploy_external_tls_galera.yaml | 22 +++ .../01-tls-certificate.yaml | 1 + .../galera_deploy_external_tls/02-assert.yaml | 8 + .../03-teardown.yaml | 16 ++ .../tests/galera_deploy_tls/01-assert.yaml | 56 ++++++ .../01-deploy_tls_galera.yaml | 22 +++ .../galera_deploy_tls/01-tls-certificate.yaml | 49 +++++ .../tests/galera_deploy_tls/02-assert.yaml | 8 + .../tests/galera_deploy_tls/03-teardown.yaml | 16 ++ 39 files changed, 1005 insertions(+), 192 deletions(-) create mode 100644 config/samples/cert-manager-galera-cert.yaml create mode 100644 config/samples/mariadb_v1beta1_galera_tls.yaml create mode 100644 templates/galera/config/galera_external_tls.cnf.in create mode 100644 templates/galera/config/galera_tls.cnf.in create mode 100644 tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml create mode 100644 tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml create mode 120000 tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml create mode 100644 tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml create mode 100644 tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml create mode 100644 tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml create mode 100644 tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml create mode 120000 tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml create mode 100644 tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml create mode 100644 tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/01-assert.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/02-assert.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index e4860d42..6c696964 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -84,6 +84,29 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + tls: + description: TLS settings for MySQL service and internal Galera replication + properties: + ca: + description: Ca contains CA-specific settings, which could be + used both by services (to define their own CA certificates) + and by clients (to verify the server's certificate) + properties: + caSecretName: + type: string + type: object + service: + description: Service contains server-specific TLS secret + properties: + disableNonTLSListeners: + type: boolean + secretName: + type: string + type: object + required: + - ca + - service + type: object required: - containerImage - replicas @@ -162,16 +185,16 @@ spec: - type type: object type: array - configHash: - default: "" - description: Hash of the configuration files - type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track input changes + type: object safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string required: - bootstrapped - - configHash type: object type: object served: true diff --git a/api/go.mod b/api/go.mod index 84f1133d..d575ae55 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,9 +4,9 @@ go 1.19 require ( github.com/go-logr/logr v1.2.4 - github.com/onsi/ginkgo/v2 v2.12.0 - github.com/onsi/gomega v1.27.10 - github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 + github.com/onsi/ginkgo/v2 v2.12.1 + github.com/onsi/gomega v1.28.0 + github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e k8s.io/api v0.26.12 k8s.io/apimachinery v0.26.12 k8s.io/client-go v0.26.12 @@ -32,7 +32,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -50,14 +50,14 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.25.0 // indirect + go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.13.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect @@ -66,7 +66,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.12 // indirect k8s.io/component-base v0.26.12 // indirect - k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/api/go.sum b/api/go.sum index 318036a2..440b22e1 100644 --- a/api/go.sum +++ b/api/go.sum @@ -41,7 +41,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -172,8 +171,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -227,14 +226,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= -github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= +github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= +github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 h1:F1iYRBwa0cZ2VHw8Zs4frqSWQ1B/tiCuSwH/DuHb8VM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0/go.mod h1:3hAC5Ce0AOSt85BqD6DgTKNkJHmpXwqbwL8mVWRJQqo= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e h1:/bKZdCAsu73wscdiMsmctAmh0Jz432WxVQe4h1+ipzQ= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e/go.mod h1:Ozg6SxfwOtMkiH553c0XQBWuygZQq4jDQCpR4hZqlxM= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -312,8 +311,8 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -480,8 +479,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -613,8 +612,8 @@ k8s.io/client-go v0.26.12 h1:kPpTpIeFNqwo4UyvoqzNp3DNK2mbGcdGv23eS1U8VMo= k8s.io/client-go v0.26.12/go.mod h1:V7thEnIFroyNZOU30dKLiiVeqQmJz45shJG1mu7nONQ= k8s.io/component-base v0.26.12 h1:OyYjCtruv4/Yau5Z1v6e59N+JRDTj8JnW95W9w9AMpg= k8s.io/component-base v0.26.12/go.mod h1:X98Et5BxJ8i4TcDusUcKS8EYxCujBU1lCL3pc/CUtHQ= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index b00981b5..98b14b69 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -69,6 +69,9 @@ const ( // MariaDBInitializedErrorMessage MariaDBInitializedErrorMessage = "MariaDB dbinit error occured %s" + // MariaDBInputSecretNotFoundMessage + MariaDBInputSecretNotFoundMessage = "Input secret not found: %s" + MariaDBDatabaseReadyInitMessage = "MariaDBDatabase not yet available" MariaDBDatabaseReadyMessage = "MariaDBDatabase ready" diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index da3abaec..4f646785 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -56,6 +57,9 @@ type GaleraSpec struct { // +kubebuilder:validation:Optional // Adoption configuration AdoptionRedirect AdoptionRedirectSpec `json:"adoptionRedirect"` + // +kubebuilder:validation:Optional + // TLS settings for MySQL service and internal Galera replication + TLS *tls.TLS `json:"tls,omitempty"` } // GaleraAttributes holds startup information for a Galera host @@ -77,9 +81,8 @@ type GaleraStatus struct { // Is the galera cluster currently running // +kubebuilder:default=false Bootstrapped bool `json:"bootstrapped"` - // Hash of the configuration files - // +kubebuilder:default="" - ConfigHash string `json:"configHash"` + // Map of hashes to track input changes + Hash map[string]string `json:"hash,omitempty"` // Deployment Conditions Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` } diff --git a/api/v1beta1/mariadbdatabase_funcs.go b/api/v1beta1/mariadbdatabase_funcs.go index 8f5e818d..7238e6f0 100644 --- a/api/v1beta1/mariadbdatabase_funcs.go +++ b/api/v1beta1/mariadbdatabase_funcs.go @@ -106,7 +106,8 @@ func (d *Database) setDatabaseHostname( err, ) } - d.databaseHostname = serviceList.Items[0].GetName() + svc := serviceList.Items[0] + d.databaseHostname = svc.GetName() + "." + svc.GetNamespace() + ".svc" return nil } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index eb1cd555..1c44c979 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -23,6 +23,7 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "k8s.io/apimachinery/pkg/runtime" ) @@ -158,6 +159,11 @@ func (in *GaleraSpec) DeepCopyInto(out *GaleraSpec) { } } out.AdoptionRedirect = in.AdoptionRedirect + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(tls.TLS) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GaleraSpec. @@ -180,6 +186,13 @@ func (in *GaleraStatus) DeepCopyInto(out *GaleraStatus) { (*out)[key] = val } } + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make(condition.Conditions, len(*in)) diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index e4860d42..6c696964 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -84,6 +84,29 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + tls: + description: TLS settings for MySQL service and internal Galera replication + properties: + ca: + description: Ca contains CA-specific settings, which could be + used both by services (to define their own CA certificates) + and by clients (to verify the server's certificate) + properties: + caSecretName: + type: string + type: object + service: + description: Service contains server-specific TLS secret + properties: + disableNonTLSListeners: + type: boolean + secretName: + type: string + type: object + required: + - ca + - service + type: object required: - containerImage - replicas @@ -162,16 +185,16 @@ spec: - type type: object type: array - configHash: - default: "" - description: Hash of the configuration files - type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track input changes + type: object safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string required: - bootstrapped - - configHash type: object type: object served: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d8dbef82..597e4d8d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -121,7 +121,9 @@ rules: - delete - get - list + - patch - update + - watch - apiGroups: - "" resources: diff --git a/config/samples/cert-manager-galera-cert.yaml b/config/samples/cert-manager-galera-cert.yaml new file mode 100644 index 00000000..0982b9a9 --- /dev/null +++ b/config/samples/cert-manager-galera-cert.yaml @@ -0,0 +1,75 @@ +# the cluster-wide issuer, used to generate a root certificate +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: {} +--- +# The root certificate. they cert/key/ca will be generated in the secret 'root-secret' +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: my-selfsigned-ca + namespace: openstack +spec: + isCA: true + commonName: my-selfsigned-ca + secretName: root-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: ClusterIssuer + group: cert-manager.io +--- +# The CA issuer for galera, uses the certificate from `my-selfsigned-ca` +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: my-ca-issuer + namespace: openstack +spec: + ca: + secretName: root-secret +--- +# The certificate used by all galera replicas for GCOMM and SST. +# The replicas in the galera statefulset all share the same +# certificate, so the latter requires wildcard in dnsNames for TLS +# validation. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: galera-cert +spec: + secretName: galera-tls + secretTemplate: + labels: + mariadb-ref: openstack + duration: 6h + renewBefore: 1h + subject: + organizations: + - cluster.local + commonName: openstack-galera + isCA: false + privateKey: + algorithm: RSA + encoding: PKCS8 + size: 2048 + usages: + - server auth + - client auth + dnsNames: + - "openstack.openstack.svc" + - "openstack.openstack.svc.cluster.local" + - "*.openstack-galera" + - "*.openstack-galera.openstack" + - "*.openstack-galera.openstack.svc" + - "*.openstack-galera.openstack.svc.cluster" + - "*.openstack-galera.openstack.svc.cluster.local" + issuerRef: + name: my-ca-issuer + group: cert-manager.io + kind: Issuer diff --git a/config/samples/mariadb_v1beta1_galera_tls.yaml b/config/samples/mariadb_v1beta1_galera_tls.yaml new file mode 100644 index 00000000..e92995d9 --- /dev/null +++ b/config/samples/mariadb_v1beta1_galera_tls.yaml @@ -0,0 +1,14 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + service: + secretName: galera-tls + ca: + caSecretName: galera-tls diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go index 59d1b28a..9e2b2afe 100644 --- a/controllers/galera_controller.go +++ b/controllers/galera_controller.go @@ -17,11 +17,13 @@ limitations under the License. package controllers import ( + "github.com/openstack-k8s-operators/lib-common/modules/common" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" configmap "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" appsv1 "k8s.io/api/apps/v1" @@ -29,6 +31,8 @@ import ( rbacv1 "k8s.io/api/rbac/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/kubectl/pkg/util/podutils" @@ -46,15 +50,33 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" databasev1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" mariadb "github.com/openstack-k8s-operators/mariadb-operator/pkg/mariadb" ) +// fields to index to reconcile on CR change +const ( + serviceSecretNameField = ".spec.tls.service.SecretName" + caSecretNameField = ".spec.tls.ca.caSecretName" +) + +var ( + allWatchFields = []string{ + serviceSecretNameField, + caSecretNameField, + } +) + // GaleraReconciler reconciles a Galera object type GaleraReconciler struct { client.Client @@ -97,7 +119,8 @@ func buildGcommURI(instance *mariadbv1.Galera) string { res := []string{} for i := 0; i < replicas; i++ { - res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename) + // Generate Gcomm with FQDN for TLS validation + res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename+"."+instance.Namespace+".svc") } uri := "gcomm://" + strings.Join(res, ",") return uri @@ -257,15 +280,17 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal if !found { continue } - ci := instance.Status.Attributes[pod.Name].ContainerID - pci := pod.Status.ContainerStatuses[0].ContainerID - if ci != pci { + // A node can have various attributes depending on its known state. + // A ContainerID attribute is only present if the node is being started. + attrCID := instance.Status.Attributes[pod.Name].ContainerID + podCID := pod.Status.ContainerStatuses[0].ContainerID + if attrCID != "" && attrCID != podCID { // This gcomm URI was pushed in a pod which was restarted // before the attribute got cleared, which means the pod // failed to start galera. Clear the attribute here, and // reprobe the pod's state in the next reconcile loop clearPodAttributes(instance, pod.Name) - util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name) + util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name, "current pod ID", podCID, "recorded ID", attrCID) } } } @@ -283,6 +308,9 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create +// RBAC for secrets +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; + // RBAC for services and endpoints // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch;create;update;patch;delete; @@ -359,6 +387,8 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res instance.Status.Conditions = condition.Conditions{} // initialize conditions used later as Status=Unknown cl := condition.CreateList( + // DB Root password and TLS secrets + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), // endpoint for adoption redirect condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), // configmap generation @@ -468,9 +498,50 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res log.Info("", "Kind", instance.Kind, "Name", instance.Name, "database service", service.Name, "operation", string(op)) } - // Generate the config maps for the various services - configMapVars := make(map[string]env.Setter) - err = r.generateConfigMaps(ctx, helper, instance, &configMapVars) + // Hash of all resources that may cause a service restart + inputHashEnv := make(map[string]env.Setter) + + // Check and hash inputs + secretName := instance.Spec.Secret + // NOTE do not hash the db root password, as its change requires + // more orchestration than a simple rolling restart + _, _, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace) + + var certHash, caHash string + tls := instance.Spec.TLS + if err == nil && tls != nil && tls.Service.SecretName != "" { + secretName := tls.Service.SecretName + _, certHash, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace) + inputHashEnv["Cert"] = env.SetValue(certHash) + } + if err == nil && tls != nil && tls.Ca.CaSecretName != "" { + secretName := tls.Ca.CaSecretName + _, caHash, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace) + inputHashEnv["CA"] = env.SetValue(caHash) + } + + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + mariadbv1.MariaDBInputSecretNotFoundMessage, + secretName)) + return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("error calculating input hash: %w", err) + } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // Generate and hash config maps + err = r.generateConfigMaps(ctx, helper, instance, &inputHashEnv) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -480,16 +551,36 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res err.Error())) return ctrl.Result{}, fmt.Errorf("error calculating configmap hash: %w", err) } - // From hereon, configMapVars holds a hash of the config generated for this instance - // This is used in an envvar in the statefulset to restart it on config change - envHash := &corev1.EnvVar{} - configMapVars[configMapNameForConfig(instance)](envHash) - instance.Status.ConfigHash = envHash.Value instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) - commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance), 5) + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + hashOfHashes, err := util.HashOfInputHashes(inputHashEnv) + if err != nil { + return ctrl.Result{}, err + } + if hashMap, changed := util.SetHash(instance.Status.Hash, common.InputHashName, hashOfHashes); changed { + // Hash changed and instance status should be updated (which will be done by main defer func), + // so update all the input hashes and return to reconcile again + instance.Status.Hash = hashMap + for k, s := range inputHashEnv { + var envVar corev1.EnvVar + s(&envVar) + instance.Status.Hash[k] = envVar.Value + } + util.LogForObject(helper, fmt.Sprintf("Input hash changed %s", hashOfHashes), instance) + return ctrl.Result{}, nil + } + + // The statefulset that manages galera pod + commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance, hashOfHashes), 5) sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper) if sferr != nil { + if k8s_errors.IsNotFound(sferr) { + return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil + } return sfres, sferr } statefulset := commonstatefulset.GetStatefulSet() @@ -662,6 +753,34 @@ func (r *GaleraReconciler) generateConfigMaps( // SetupWithManager sets up the controller with the Manager. func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { r.config = mgr.GetConfig() + + // Various CR fields need to be indexed to filter watch events + // for the secret changes we want to be notified of + // index caSecretName + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &mariadbv1.Galera{}, caSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*mariadbv1.Galera) + tls := cr.Spec.TLS + if tls != nil && tls.Ca != nil && tls.Ca.CaSecretName == "" { + return []string{tls.Ca.CaSecretName} + } + return nil + }); err != nil { + return err + } + // index serviceSecretName + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &mariadbv1.Galera{}, serviceSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*mariadbv1.Galera) + tls := cr.Spec.TLS + if tls != nil && tls.Service != nil && tls.Service.SecretName == "" { + return []string{cr.Spec.TLS.Service.SecretName} + } + return nil + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&mariadbv1.Galera{}). Owns(&appsv1.StatefulSet{}). @@ -671,6 +790,11 @@ func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } @@ -715,3 +839,33 @@ func GetDatabaseObject(clientObj client.Client, ctx context.Context, name string return dbGalera, nil } + +// findObjectsForSrc - returns a reconcile request if the object is referenced by a Galera CR +func (r *GaleraReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + for _, field := range allWatchFields { + crList := &mariadbv1.GaleraList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} diff --git a/controllers/mariadbdatabase_controller.go b/controllers/mariadbdatabase_controller.go index 92430262..1ce43ee6 100644 --- a/controllers/mariadbdatabase_controller.go +++ b/controllers/mariadbdatabase_controller.go @@ -132,6 +132,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Non-deletion (normal) flow follows // var dbName, dbSecret, dbContainerImage, serviceAccount string + var useTLS bool // It is impossible to reach here without either dbGalera or dbMariadb not being nil, due to the checks above @@ -149,6 +150,11 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ dbSecret = dbGalera.Spec.Secret dbContainerImage = dbGalera.Spec.ContainerImage serviceAccount = dbGalera.RbacResourceName() + // NOTE(dciabrin) When configured to only allow TLS connections, all clients + // accessing this DB must support client connection via TLS. + useTLS = (dbGalera.Spec.TLS != nil && + dbGalera.Spec.TLS.Service != nil && + dbGalera.Spec.TLS.Service.DisableNonTLSListeners) } else if isMariaDB { if dbMariadb.Status.DbInitHash == "" { log.Info("DB initialization not complete. Requeue...") @@ -159,10 +165,11 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ dbSecret = dbMariadb.Spec.Secret dbContainerImage = dbMariadb.Spec.ContainerImage serviceAccount = dbMariadb.RbacResourceName() + useTLS = false } // Define a new Job object (hostname, password, containerImage) - jobDef, err := mariadb.DbDatabaseJob(instance, dbName, dbSecret, dbContainerImage, serviceAccount) + jobDef, err := mariadb.DbDatabaseJob(instance, dbName, dbSecret, dbContainerImage, serviceAccount, useTLS) if err != nil { return ctrl.Result{}, err } diff --git a/go.mod b/go.mod index fbba3d53..087c55d1 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.19 require ( github.com/go-logr/logr v1.2.4 github.com/onsi/ginkgo/v2 v2.12.1 - github.com/onsi/gomega v1.27.10 - github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230927082538-4f614f333d17 + github.com/onsi/gomega v1.28.0 + github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e github.com/openstack-k8s-operators/mariadb-operator/api v0.1.1-0.20230823144333-b9363c5be8d2 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 k8s.io/api v0.26.12 diff --git a/go.sum b/go.sum index 304b7186..0906bee0 100644 --- a/go.sum +++ b/go.sum @@ -231,10 +231,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230927082538-4f614f333d17 h1:n5QmZLJfPtKbNnPVqqSQkLU1X/NMmW3CbML3yjBUjyY= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230927082538-4f614f333d17/go.mod h1:kZS5rqVWBZeCyYor2PeQB9IEZ19mGaeL/to3x8F9OJg= +github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= +github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e h1:/bKZdCAsu73wscdiMsmctAmh0Jz432WxVQe4h1+ipzQ= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e/go.mod h1:Ozg6SxfwOtMkiH553c0XQBWuygZQq4jDQCpR4hZqlxM= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/pkg/mariadb/database.go b/pkg/mariadb/database.go index 59b45719..59980d20 100644 --- a/pkg/mariadb/database.go +++ b/pkg/mariadb/database.go @@ -16,10 +16,17 @@ type dbCreateOptions struct { DatabaseAdminUsername string DefaultCharacterSet string DefaultCollation string + DatabaseUserTLS string } // DbDatabaseJob - -func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string) (*batchv1.Job, error) { +func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, useTLS bool) (*batchv1.Job, error) { + var tlsStatement string + if useTLS { + tlsStatement = " REQUIRE SSL" + } else { + tlsStatement = "" + } opts := dbCreateOptions{ database.Spec.Name, @@ -27,6 +34,7 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s "root", database.Spec.DefaultCharacterSet, database.Spec.DefaultCollation, + tlsStatement, } dbCmd, err := util.ExecuteTemplateFile("database.sh", &opts) if err != nil { @@ -97,6 +105,7 @@ func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHost "root", database.Spec.DefaultCharacterSet, database.Spec.DefaultCollation, + "", } delCmd, err := util.ExecuteTemplateFile("delete_database.sh", &opts) if err != nil { diff --git a/pkg/mariadb/statefulset.go b/pkg/mariadb/statefulset.go index 2abcfff8..b945e555 100644 --- a/pkg/mariadb/statefulset.go +++ b/pkg/mariadb/statefulset.go @@ -11,9 +11,12 @@ import ( ) // StatefulSet returns a StatefulSet object for the galera cluster -func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { +func StatefulSet(g *mariadbv1.Galera, configHash string) *appsv1.StatefulSet { ls := StatefulSetLabels(g) name := StatefulSetName(g.Name) + replicas := g.Spec.Replicas + storage := g.Spec.StorageClass + storageRequest := resource.MustParse(g.Spec.StorageRequest) sts := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -21,7 +24,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, Spec: appsv1.StatefulSetSpec{ ServiceName: name, - Replicas: g.Spec.Replicas, + Replicas: replicas, Selector: &metav1.LabelSelector{ MatchLabels: ls, }, @@ -53,25 +56,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraInitVolumeMounts(g), }}, Containers: []corev1.Container{{ Image: g.Spec.ContainerImage, @@ -80,7 +65,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { Command: []string{"/usr/bin/dumb-init", "--", "/usr/local/bin/kolla_start"}, Env: []corev1.EnvVar{{ Name: "CR_CONFIG_HASH", - Value: g.Status.ConfigHash, + Value: configHash, }, { Name: "KOLLA_CONFIG_STRATEGY", Value: "COPY_ALWAYS", @@ -102,29 +87,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { ContainerPort: 4567, Name: "galera", }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/secrets", - ReadOnly: true, - Name: "secrets", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraVolumeMounts(g), StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ @@ -149,92 +112,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - Volumes: []corev1.Volume{ - { - Name: "secrets", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: g.Spec.Secret, - Items: []corev1.KeyToPath{ - { - Key: "DbRootPassword", - Path: "dbpassword", - }, - }, - }, - }, - }, - { - Name: "kolla-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "pod-config-data", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "config-data", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "galera.cnf.in", - Path: "galera.cnf.in", - }, - { - Key: mariadbv1.CustomServiceConfigFile, - Path: mariadbv1.CustomServiceConfigFile, - }, - }, - }, - }, - }, - { - Name: "operator-scripts", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-scripts", - }, - Items: []corev1.KeyToPath{ - { - Key: "mysql_bootstrap.sh", - Path: "mysql_bootstrap.sh", - }, - { - Key: "mysql_probe.sh", - Path: "mysql_probe.sh", - }, - { - Key: "detect_last_commit.sh", - Path: "detect_last_commit.sh", - }, - { - Key: "detect_gcomm_and_start.sh", - Path: "detect_gcomm_and_start.sh", - }, - }, - }, - }, - }, - }, + Volumes: getGaleraVolumes(g), }, }, VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ @@ -247,10 +125,10 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, - StorageClassName: &g.Spec.StorageClass, + StorageClassName: &storage, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - "storage": resource.MustParse(g.Spec.StorageRequest), + "storage": storageRequest, }, }, }, diff --git a/pkg/mariadb/volumes.go b/pkg/mariadb/volumes.go index 459131f3..a6f2cbb5 100644 --- a/pkg/mariadb/volumes.go +++ b/pkg/mariadb/volumes.go @@ -1,6 +1,7 @@ package mariadb import ( + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" ) @@ -113,3 +114,181 @@ func getInitVolumeMounts() []corev1.VolumeMount { } } + +func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { + configTemplates := []corev1.KeyToPath{ + { + Key: "galera.cnf.in", + Path: "galera.cnf.in", + }, + { + Key: mariadbv1.CustomServiceConfigFile, + Path: mariadbv1.CustomServiceConfigFile, + }, + } + + if g.Spec.TLS != nil && g.Spec.TLS.Service.SecretName != "" { + if g.Spec.TLS.Ca.CaSecretName != "" { + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_tls.cnf.in", + Path: "galera_tls.cnf.in", + }) + } else { + // Without a CA, WSREP is unencrypted. Only SQL traffic is. + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_external_tls.cnf.in", + Path: "galera_external_tls.cnf.in", + }) + } + } + + volumes := []corev1.Volume{ + { + Name: "secrets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: g.Spec.Secret, + Items: []corev1.KeyToPath{ + { + Key: "DbRootPassword", + Path: "dbpassword", + }, + }, + }, + }, + }, + { + Name: "kolla-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: []corev1.KeyToPath{ + { + Key: "config.json", + Path: "config.json", + }, + }, + }, + }, + }, + { + Name: "pod-config-data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: configTemplates, + }, + }, + }, + { + Name: "operator-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-scripts", + }, + Items: []corev1.KeyToPath{ + { + Key: "mysql_bootstrap.sh", + Path: "mysql_bootstrap.sh", + }, + { + Key: "mysql_probe.sh", + Path: "mysql_probe.sh", + }, + { + Key: "detect_last_commit.sh", + Path: "detect_last_commit.sh", + }, + { + Key: "detect_gcomm_and_start.sh", + Path: "detect_gcomm_and_start.sh", + }, + }, + }, + }, + }, + } + + if g.Spec.TLS != nil { + caVolumes := g.Spec.TLS.CreateVolumes() + volumes = append(volumes, caVolumes...) + } + + return volumes +} + +func getGaleraVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data", + ReadOnly: true, + Name: "config-data", + }, { + MountPath: "/var/lib/pod-config-data", + Name: "pod-config-data", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { + MountPath: "/var/lib/kolla/config_files", + ReadOnly: true, + Name: "kolla-config", + }, + } + + if g.Spec.TLS != nil { + caVolumeMounts := g.Spec.TLS.CreateVolumeMounts() + volumeMounts = append(volumeMounts, caVolumeMounts...) + } + + return volumeMounts +} + +func getGaleraInitVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data", + ReadOnly: true, + Name: "config-data", + }, { + MountPath: "/var/lib/pod-config-data", + Name: "pod-config-data", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { + MountPath: "/var/lib/kolla/config_files", + ReadOnly: true, + Name: "kolla-config", + }, + } + + return volumeMounts +} diff --git a/templates/database.sh b/templates/database.sh index 96484af9..7b933395 100755 --- a/templates/database.sh +++ b/templates/database.sh @@ -1,4 +1,4 @@ #!/bin/bash export DatabasePassword=${DatabasePassword:?"Please specify a DatabasePassword variable."} -mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; ALTER DATABASE {{.DatabaseName}} CHARACTER SET '{{.DefaultCharacterSet}}' COLLATE '{{.DefaultCollation}}'; GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'localhost' IDENTIFIED BY '$DatabasePassword';GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'%' IDENTIFIED BY '$DatabasePassword';" +mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; ALTER DATABASE {{.DatabaseName}} CHARACTER SET '{{.DefaultCharacterSet}}' COLLATE '{{.DefaultCollation}}'; GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'localhost' IDENTIFIED BY '$DatabasePassword'{{.DatabaseUserTLS}};GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'%' IDENTIFIED BY '$DatabasePassword'{{.DatabaseUserTLS}};" diff --git a/templates/galera/config/config.json b/templates/galera/config/config.json index 185e33fa..bc2b6f33 100644 --- a/templates/galera/config/config.json +++ b/templates/galera/config/config.json @@ -7,6 +7,20 @@ "owner": "root", "perm": "0644" }, + { + "source": "/var/lib/pod-config-data/galera_tls.cnf", + "dest": "/etc/my.cnf.d/galera_tls.cnf", + "owner": "root", + "perm": "0644", + "optional": true + }, + { + "source": "/var/lib/pod-config-data/galera_external_tls.cnf", + "dest": "/etc/my.cnf.d/galera_external_tls.cnf", + "owner": "root", + "perm": "0644", + "optional": true + }, { "source": "/var/lib/pod-config-data/galera_custom.cnf", "dest": "/etc/my.cnf.d/galera_custom.cnf", @@ -20,6 +34,27 @@ "owner": "root", "perm": "0755", "merge": "true" + }, + { + "source": "/var/lib/config-data/tls-certificates/tls.key", + "dest": "/etc/pki/tls/private/mysql.key", + "owner": "mysql", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/config-data/tls-certificates/tls.crt", + "dest": "/etc/pki/tls/certs/mysql.crt", + "owner": "mysql", + "perm": "0755", + "optional": true + }, + { + "source": "/var/lib/config-data/ca-certificates/ca.crt", + "dest": "/etc/ipa/ca.crt", + "owner": "mysql", + "perm": "0444", + "optional": true } ], "permissions": [ diff --git a/templates/galera/config/galera_external_tls.cnf.in b/templates/galera/config/galera_external_tls.cnf.in new file mode 100644 index 00000000..20a099f0 --- /dev/null +++ b/templates/galera/config/galera_external_tls.cnf.in @@ -0,0 +1,8 @@ +[mysqld] +ssl +ssl-cert = /etc/pki/tls/certs/mysql.crt +ssl-key = /etc/pki/tls/private/mysql.key +ssl-cipher = !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 + +[sst] +ssl-mode = DISABLED diff --git a/templates/galera/config/galera_tls.cnf.in b/templates/galera/config/galera_tls.cnf.in new file mode 100644 index 00000000..040c4e60 --- /dev/null +++ b/templates/galera/config/galera_tls.cnf.in @@ -0,0 +1,15 @@ +[mysqld] +ssl +ssl-cert = /etc/pki/tls/certs/mysql.crt +ssl-key = /etc/pki/tls/private/mysql.key +ssl-ca = /etc/ipa/ca.crt +ssl-cipher = !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +wsrep_provider_options = gcache.recover=no;gmcast.listen_addr=tcp://{ PODIP }:4567;socket.ssl_key=/etc/pki/tls/private/mysql.key;socket.ssl_cert=/etc/pki/tls/certs/mysql.crt;socket.ssl_cipher=AES128-SHA256;socket.ssl_ca=/etc/ipa/ca.crt; + +[sst] +sockopt = cipher=!SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +tcert = /etc/pki/tls/certs/mysql.crt +tkey = /etc/pki/tls/private/mysql.key +tca = /etc/ipa/ca.crt +encrypt = 3 +ssl-mode = REQUIRED diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index 33d9720c..2df58a6c 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -28,6 +28,10 @@ status: reason: Ready status: "True" type: ExposeServiceReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml new file mode 100644 index 00000000..4e0eaec3 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml @@ -0,0 +1,44 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 1 + secret: osp-secret + storageRequest: 500M + tls: + service: + secretName: kuttl-galera-tls +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml new file mode 100644 index 00000000..d94f3ba8 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml @@ -0,0 +1,45 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBDatabase + name: kuttldb + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource with external TLS only (no TLS for galera WSREP traffic) +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 1 + tls: + service: + secretName: kuttl-galera-tls + disableNonTLSListeners: true + ca: + caSecretName: kuttl-galera-tls +--- +# a database cr associated with the db above will create users with grants +# set up to only allow TLS connection +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + name: kuttldb + labels: + dbName: openstack +spec: + secret: kuttldb-secret + name: kuttldb +--- +apiVersion: v1 +kind: Secret +metadata: + name: kuttldb-secret +data: + DatabasePassword: MTIzNDU2Nzg= diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml new file mode 120000 index 00000000..d04979c5 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml @@ -0,0 +1 @@ +../galera_deploy_tls/01-tls-certificate.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml b/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml new file mode 100644 index 00000000..ecb23685 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure db users are configured to TLS restriction + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -e "show grants for \`kuttldb\`@\`%\`;"' | grep 'REQUIRE SSL' diff --git a/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml b/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml new file mode 100644 index 00000000..0d81c742 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBDatabase + name: kuttldb + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml b/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml index 5d073a88..d87f62c4 100644 --- a/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml +++ b/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml @@ -7,3 +7,4 @@ delete: commands: - script: | oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml new file mode 100644 index 00000000..7b80b88b --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml @@ -0,0 +1,54 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M + tls: + service: + secretName: kuttl-galera-tls +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-2 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml new file mode 100644 index 00000000..902f6d35 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource with external TLS only (no TLS for galera WSREP traffic) +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + service: + secretName: kuttl-galera-tls + ca: + caSecretName: diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml new file mode 120000 index 00000000..d04979c5 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml @@ -0,0 +1 @@ +../galera_deploy_tls/01-tls-certificate.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml b/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml new file mode 100644 index 00000000..e59da5b7 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure galera does not encrypt WSREP traffic + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.wsrep_provider_options;"' | grep -o -w 'gmcast.listen_addr = tcp' + # ensure mysql/SQL traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.ssl_cipher;"' | grep -v '^NULL$' diff --git a/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml b/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml new file mode 100644 index 00000000..68c7f897 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml b/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml new file mode 100644 index 00000000..ae35e3e2 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml @@ -0,0 +1,56 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M + tls: + service: + secretName: kuttl-galera-tls + ca: + caSecretName: kuttl-galera-tls +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-2 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml b/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml new file mode 100644 index 00000000..870aaa22 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource using the TLS certs +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + service: + secretName: kuttl-galera-tls + ca: + caSecretName: kuttl-galera-tls diff --git a/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml new file mode 100644 index 00000000..f50c4cde --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml @@ -0,0 +1,49 @@ +# hardcode the secret generated by a certificate CR here, +# to avoid kuttl from depending on cert-manager at runtime +# --- +# apiVersion: cert-manager.io/v1 +# kind: Certificate +# metadata: +# name: kuttl-galera-cert +# spec: +# secretName: kuttl-galera-tls +# secretTemplate: +# labels: +# mariadb-ref: openstack +# duration: 12720h +# renewBefore: 1h +# subject: +# organizations: +# - cluster.local +# commonName: openstack-galera +# isCA: false +# privateKey: +# algorithm: RSA +# encoding: PKCS8 +# size: 1024 +# usages: +# - server auth +# - client auth +# dnsNames: +# - "openstack.openstack.svc" +# - "openstack.openstack.svc.cluster.local" +# - "*.openstack-galera" +# - "*.openstack-galera.openstack" +# - "*.openstack-galera.openstack.svc" +# - "*.openstack-galera.openstack.svc.cluster" +# - "*.openstack-galera.openstack.svc.cluster.local" +# issuerRef: +# name: kuttl-ca-issuer +# group: cert-manager.io +# kind: Issuer +# --- +apiVersion: v1 +kind: Secret +metadata: + name: kuttl-galera-tls + labels: + mariadb-ref: openstack +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJmVENDQVNLZ0F3SUJBZ0lRUHhtRFFscmxjNTNhb215RVU5MU9pakFLQmdncWhrak9QUVFEQWpBZU1Sd3cKR2dZRFZRUURFeE5yZFhSMGJDMXpaV3htYzJsbmJtVmtMV05oTUI0WERUSXpNVEF4T0RFeU1EazFNMW9YRFRJMApNREV4TmpFeU1EazFNMW93SGpFY01Cb0dBMVVFQXhNVGEzVjBkR3d0YzJWc1puTnBaMjVsWkMxallUQlpNQk1HCkJ5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCSVdJY0JiR0cveEg4Lzlkc2lMbkJCdnRqcEZoQ2JRM3U4R0EKZXBVcnhTY25XM0hrZ2hrc1BCVE12M3NCeGdnVFQwL0Eva0dtazRYTkJ0dElnbUZJaFBpalFqQkFNQTRHQTFVZApEd0VCL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlFKaDd3VklFYjgxcFlsCkl3RDAraTBwSnlCTjNqQUtCZ2dxaGtqT1BRUURBZ05KQURCR0FpRUF2a3h5RzZjNzltSDlRWHRIVWFSM014REkKUUVRRGVtL1hZR3VGY1ZCUDJpQUNJUUNFeEZqeStQUTBkNFU5dEJacTVOd1gzdmxibnQxVlNCYWE5VFIrNkNkbAozdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhekNDQXhDZ0F3SUJBZ0lRUWw2bjhlS21BbXZlbDhZTGhIaFVjREFLQmdncWhrak9QUVFEQWpBZU1Sd3cKR2dZRFZRUURFeE5yZFhSMGJDMXpaV3htYzJsbmJtVmtMV05oTUI0WERUSXpNVEF5TXpBNE16UTBNRm9YRFRJMQpNRFF3TlRBNE16UTBNRm93TXpFV01CUUdBMVVFQ2hNTlkyeDFjM1JsY2k1c2IyTmhiREVaTUJjR0ExVUVBeE1RCmIzQmxibk4wWVdOckxXZGhiR1Z5WVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUIKQU9QMnFUVkhvUmFsb1VCb0RSS1NrZExsR2ZqV2U2WEFFQ21HZGpRdzNjY0NHWUNKc3U3akROSFMxalJCa0ZUKwpnVERUNGk5MndPelI2WGpWQ2Fld1pCSXlyU1d4N3Y3SndaZFkvVkQzVWdRSFdpWEpNOFJlRFZUV0J6TjdqL3JTCktGY0lIcURGYjFGRy9jS0gzcEJrTVRReTZSZXRRdHRkTVl0bE1tVEdFdFJnQVZQdGlLTXZnMXZadThUY053VEwKZ2h0cjFOdURBYzFpVXhzL1p5S0o0ZjBHNEJMVlNHNVgwRXRvVUZxcWF3R2EySGhMZld5TXEyRWdoWGE1QzZHeQoxSkdVYUtCZHdCMjRLd2loSVR0U1Z3SGxrK2FGQ2Nac3FzT1BuR2c4WTZRSkM1bERUeE1UWXpHN1pxaGtMVmgwClFJTUpVMnNtMEd3aENYWVlGOXF5c2JNQ0F3RUFBYU9DQVU0d2dnRktNQjBHQTFVZEpRUVdNQlFHQ0NzR0FRVUYKQndNQkJnZ3JCZ0VGQlFjREFqQU1CZ05WSFJNQkFmOEVBakFBTUI4R0ExVWRJd1FZTUJhQUZBbUh2QlVnUnZ6VwpsaVVqQVBUNkxTa25JRTNlTUlINUJnTlZIUkVFZ2ZFd2dlNkNGMjl3Wlc1emRHRmpheTV2Y0dWdWMzUmhZMnN1CmMzWmpnaVZ2Y0dWdWMzUmhZMnN1YjNCbGJuTjBZV05yTG5OMll5NWpiSFZ6ZEdWeUxteHZZMkZzZ2hJcUxtOXcKWlc1emRHRmpheTFuWVd4bGNtR0NIQ291YjNCbGJuTjBZV05yTFdkaGJHVnlZUzV2Y0dWdWMzUmhZMnVDSUNvdQpiM0JsYm5OMFlXTnJMV2RoYkdWeVlTNXZjR1Z1YzNSaFkyc3VjM1pqZ2lncUxtOXdaVzV6ZEdGamF5MW5ZV3hsCmNtRXViM0JsYm5OMFlXTnJMbk4yWXk1amJIVnpkR1Z5Z2k0cUxtOXdaVzV6ZEdGamF5MW5ZV3hsY21FdWIzQmwKYm5OMFlXTnJMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNNQW9HQ0NxR1NNNDlCQU1DQTBrQU1FWUNJUUNIbllHMApxWFhqVGxPSFFpaXpwdVN0NVVQc3F5UXY3eTVNVDhaZ3RxYmd3Z0loQVBRdUNWeFBMb3daOGVWV3BTc09DVnNICjM0ZmFhWFFmbjVEZXE0eGFGSXFmCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRRGo5cWsxUjZFV3BhRkEKYUEwU2twSFM1Um40MW51bHdCQXBoblkwTU4zSEFobUFpYkx1NHd6UjB0WTBRWkJVL29FdzArSXZkc0RzMGVsNAoxUW1uc0dRU01xMGxzZTcreWNHWFdQMVE5MUlFQjFvbHlUUEVYZzFVMWdjemU0LzYwaWhYQ0I2Z3hXOVJSdjNDCmg5NlFaREUwTXVrWHJVTGJYVEdMWlRKa3hoTFVZQUZUN1lpakw0TmIyYnZFM0RjRXk0SWJhOVRiZ3dITllsTWIKUDJjaWllSDlCdUFTMVVodVY5QkxhRkJhcW1zQm10aDRTMzFzakt0aElJVjJ1UXVoc3RTUmxHaWdYY0FkdUNzSQpvU0U3VWxjQjVaUG1oUW5HYktyRGo1eG9QR09rQ1F1WlEwOFRFMk14dTJhb1pDMVlkRUNEQ1ZOckp0QnNJUWwyCkdCZmFzckd6QWdNQkFBRUNnZ0VBQlBwQzBaWUJ6M1BJWkphaENuSEcyMXNEU1NxdHduM0NISllDQ0x1MGEvOEEKdll6RUtuRXN1M3dtRHpnTFJiL1U1Rk1NRWtzK2hTMUVNczQ4Q3J4UVF4RjJOR2VCTjNFQ1dDS3FUVlZKSml0ZQo3WXBLSmhTRHFBR1FZTEJuQVd3TFUzTXE4QW0vbDMrOFZMNFF2d2hoQnFuQjZ6RnVUMG5tcGd1TW9TU3VEcE1MCmh3akN6QlU3cHJ1UEJIOVNrWHNNKzRGeHVkOTlJMUo4YkdDeWN5Qm9OaFc4OHgwYVJ2ZU9DdVVXaHVHdjBCT3oKYkRkZXE2cEI0S0hjL3QyUmg4S05lcjVETE9YdDBaeU96a3VMWEFQM2xzTE9mOHBOWWtHVjJ0dzBJQ3pkMndmNQphSTk4c2FDSDRuQXUyTHBEUCtkM0xmTXlML2EwamszOFVpbUs5WXBTZ1FLQmdRRHNndi91WGw2bTR3Qm1LemphCitLNzVXLzlnc25pK3U3MXl2NkhwZ3l3T2FsTWxIczV5bTg3blhoUFJtc3FyQlhBeG5lNW1Sb29TWjVhKzlLak8Ka3IwT1VXSnkrb2JQbFdmdUtVTXdLbmplVy93UUsrWlpOTGJlM0ZsRHVxM2NOczRGMHFXN2J2dHN0MURXekl4QQp5WHZUaEhxL3ltdGlDMXBYVENHZHN2bXV3UUtCZ1FEMnYxaEJOT09Ea1NRb1ZGZzJLQlplYmNWUzFEUjZ2VVcvCjByMEIyNjRYRzVFRXIxSkNabFUxRG1jR1c2ZHowVkdQeWt6VXhaa2hmZmV2SHdoNXBUZStnVndEd0huMjRFTkkKV1NIaytzbHhBUlFMbFVRakpEczdvaVhqK0ZZcjJhd0JrTWxjVjN4cEorN2syYTFaMVlHZnVmait3Z2FuUStDRwpJUXJaZmVaeGN3S0JnUUNFSXlCb0JkTnQrKzl5SlgzTFlSc28rQXd5OHlOZ3RMVkFrZTRjNzR0Q3RvYWplNVd1CkZIekJhUjg3Z1Bid0c4YTJBZDE2eUxoRlZoaXpzUzlLMGxMTDJBNWYzTFpLN2RjMkNWbWZaR3RKR0MwNHY3NGYKWXNNMHVma3BUUGZyeTdiSkxBb3FNUFJKcGhXRkhKelRhWDVFQzRVaytDdU1pSGs5d3F3WlZqQUhRUUtCZ1FEeQpVZDJ4UStoL1o4V1RtYWw3bmdqQnVabkVpMFB5bWZocXR1SUpkaDFVOGl5TDVrV3BrZWZ5RCsvYmpaMDRuNE15CjdDQmlBS0F2QlE5K09zTE8vQk52a29pejRvMmtobjl0Q0tQQnhReVpYczY1eFdCQ0JxM3BGWWVTaDJyUTY1Y0IKZnBoUmlGVlV5L1BQYVhyVnMxajJiT1A4Qkw0Vi9CSCtoMnFwUnZ4Tnd3S0JnQk82c3dJeURBYmFKc0lOS2ZBZQp3R3g5d0tkcVUyM052TG9oaEFkTVpoZ0dEcHM0Sm13TEJxYTZUb05NSzVLQUZMQzJsenBWT1RvWlZCQUpNVWFpCmlVblJmT0hoMjhhQWhGdHlTZ3dNcWg2M2xqWnNuWXEzdmVzQmJoSTVYQWhDY2xNUVR4b0k2RDhUNTZEWmNEY04KZGx3bkVJamFMV205aE5XMXlPQ0EvUXFVCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml b/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml new file mode 100644 index 00000000..cd069330 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure galera/WSREP traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.wsrep_provider_options;"' | grep -o -w 'socket.ssl = YES' + # ensure mysql/SQL traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.ssl_cipher;"' | grep -v '^NULL$' diff --git a/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml b/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml new file mode 100644 index 00000000..68c7f897 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done