From 448c5981a7b5f0848f4267b55b0a4076d5e022b4 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Wed, 13 Dec 2023 11:19:04 +0100 Subject: [PATCH] [tlse] tls for NovaAPI, NovaMetadata and NovNoVNCProxy Public/Internal service cert secrets and the CA bundle secret can be passed to configure httpd virtual hosts for tls termination. The CA cert get direct mounted as the environment bundle to /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem . The service certificates like config files and copied via kolla to /etc/pki/tls/certs/%s.crt|/etc/pki/tls/private/%s.key . Job deployments for bootstrap/cron get the CA bundle added if configured. Also indexes the named input resources for CA bundle, and tls secrets to be able to watch them for a change and reconcile. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/384 Jira: TODO --- api/bases/nova.openstack.org_nova.yaml | 65 ++++++ api/bases/nova.openstack.org_novaapis.yaml | 30 +++ api/bases/nova.openstack.org_novacells.yaml | 30 +++ .../nova.openstack.org_novacomputes.yaml | 8 + .../nova.openstack.org_novaconductors.yaml | 8 + .../nova.openstack.org_novametadata.yaml | 11 ++ .../nova.openstack.org_novanovncproxies.yaml | 11 ++ .../nova.openstack.org_novaschedulers.yaml | 8 + api/go.mod | 2 + api/go.sum | 4 +- api/v1beta1/novaapi_types.go | 11 ++ api/v1beta1/novacell_types.go | 6 + api/v1beta1/novacompute_types.go | 7 + api/v1beta1/novaconductor_types.go | 7 + api/v1beta1/novametadata_types.go | 12 ++ api/v1beta1/novanovncproxy_types.go | 12 ++ api/v1beta1/novascheduler_types.go | 6 + api/v1beta1/zz_generated.deepcopy.go | 10 + config/crd/bases/nova.openstack.org_nova.yaml | 65 ++++++ .../bases/nova.openstack.org_novaapis.yaml | 30 +++ .../bases/nova.openstack.org_novacells.yaml | 30 +++ .../nova.openstack.org_novacomputes.yaml | 8 + .../nova.openstack.org_novaconductors.yaml | 8 + .../nova.openstack.org_novametadata.yaml | 11 ++ .../nova.openstack.org_novanovncproxies.yaml | 11 ++ .../nova.openstack.org_novaschedulers.yaml | 8 + controllers/nova_controller.go | 4 + controllers/novaapi_controller.go | 185 +++++++++++++++++- controllers/novacompute_controller.go | 101 ++++++++++ controllers/novaconductor_controller.go | 101 ++++++++++ controllers/novametadata_controller.go | 165 +++++++++++++++- controllers/novanovncproxy_controller.go | 154 ++++++++++++++- controllers/novascheduler_controller.go | 101 ++++++++++ go.mod | 2 + go.sum | 4 +- pkg/nova/cellmapping.go | 29 ++- pkg/nova/host_discover.go | 29 ++- pkg/novaapi/deployment.go | 61 ++++-- pkg/novacompute/deployment.go | 26 ++- pkg/novaconductor/dbsync.go | 30 ++- pkg/novaconductor/deployment.go | 26 ++- pkg/novametadata/deployment.go | 55 ++++-- pkg/novascheduler/deployment.go | 26 ++- pkg/novncproxy/deployment.go | 45 ++++- templates/nova.conf | 15 +- templates/novaapi/config/httpd.conf | 40 +++- templates/novaapi/config/nova-api-config.json | 22 +++ templates/novaapi/config/ssl.conf | 21 ++ templates/novametadata/config/httpd.conf | 16 ++ .../config/nova-metadata-config.json | 32 ++- templates/novametadata/config/ssl.conf | 21 ++ .../config/nova-novncproxy-config.json | 16 ++ test/functional/base_test.go | 16 ++ .../nova_compute_ironic_controller_test.go | 62 ++++++ .../nova_metadata_controller_test.go | 100 ++++++++++ test/functional/nova_novncproxy_test.go | 95 +++++++++ test/functional/nova_scheduler_test.go | 57 ++++++ test/functional/novaapi_controller_test.go | 141 +++++++++++++ .../novaconductor_controller_test.go | 59 ++++++ 59 files changed, 2156 insertions(+), 120 deletions(-) create mode 100644 templates/novaapi/config/ssl.conf create mode 100644 templates/novametadata/config/ssl.conf diff --git a/api/bases/nova.openstack.org_nova.yaml b/api/bases/nova.openstack.org_nova.yaml index 6fefa0194..7fe1fbb3c 100644 --- a/api/bases/nova.openstack.org_nova.yaml +++ b/api/bases/nova.openstack.org_nova.yaml @@ -326,6 +326,36 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object type: object cellTemplates: additionalProperties: @@ -745,6 +775,18 @@ spec: value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object type: object noVNCProxyServiceTemplate: description: NoVNCProxyServiceTemplate - defines the novncproxy @@ -1040,6 +1082,18 @@ spec: value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object type: object nodeSelector: additionalProperties: @@ -1497,6 +1551,17 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object type: object nodeSelector: additionalProperties: diff --git a/api/bases/nova.openstack.org_novaapis.yaml b/api/bases/nova.openstack.org_novaapis.yaml index e520a11b8..ef45bc78b 100644 --- a/api/bases/nova.openstack.org_novaapis.yaml +++ b/api/bases/nova.openstack.org_novaapis.yaml @@ -383,6 +383,36 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - apiDatabaseHostname - cell0DatabaseHostname diff --git a/api/bases/nova.openstack.org_novacells.yaml b/api/bases/nova.openstack.org_novacells.yaml index 0170ac22c..6a7bc9b6d 100644 --- a/api/bases/nova.openstack.org_novacells.yaml +++ b/api/bases/nova.openstack.org_novacells.yaml @@ -455,6 +455,17 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object type: object noVNCProxyServiceTemplate: description: NoVNCProxyServiceTemplate - defines the novvncproxy service @@ -729,6 +740,17 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object type: object nodeSelector: additionalProperties: @@ -861,6 +883,14 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - cellDatabaseHostname - cellName diff --git a/api/bases/nova.openstack.org_novacomputes.yaml b/api/bases/nova.openstack.org_novacomputes.yaml index 271191c96..e40199c89 100644 --- a/api/bases/nova.openstack.org_novacomputes.yaml +++ b/api/bases/nova.openstack.org_novacomputes.yaml @@ -189,6 +189,14 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - cellName - computeDriver diff --git a/api/bases/nova.openstack.org_novaconductors.yaml b/api/bases/nova.openstack.org_novaconductors.yaml index 1371c5a44..a588a3da4 100644 --- a/api/bases/nova.openstack.org_novaconductors.yaml +++ b/api/bases/nova.openstack.org_novaconductors.yaml @@ -201,6 +201,14 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - cellName - keystoneAuthURL diff --git a/api/bases/nova.openstack.org_novametadata.yaml b/api/bases/nova.openstack.org_novametadata.yaml index d18df7ebb..77cc99e42 100644 --- a/api/bases/nova.openstack.org_novametadata.yaml +++ b/api/bases/nova.openstack.org_novametadata.yaml @@ -378,6 +378,17 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object required: - keystoneAuthURL - secret diff --git a/api/bases/nova.openstack.org_novanovncproxies.yaml b/api/bases/nova.openstack.org_novanovncproxies.yaml index fb77c4c9b..3073a61ae 100644 --- a/api/bases/nova.openstack.org_novanovncproxies.yaml +++ b/api/bases/nova.openstack.org_novanovncproxies.yaml @@ -356,6 +356,17 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object required: - cellDatabaseHostname - cellName diff --git a/api/bases/nova.openstack.org_novaschedulers.yaml b/api/bases/nova.openstack.org_novaschedulers.yaml index 4fe449479..d169f8993 100644 --- a/api/bases/nova.openstack.org_novaschedulers.yaml +++ b/api/bases/nova.openstack.org_novaschedulers.yaml @@ -204,6 +204,14 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - apiDatabaseHostname - cell0DatabaseHostname diff --git a/api/go.mod b/api/go.mod index 11bf0a8f2..d23f47988 100644 --- a/api/go.mod +++ b/api/go.mod @@ -68,3 +68,5 @@ require ( // mschuppert: map to latest commit from release-4.13 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/deydra71/lib-common/modules/common v0.0.0-20240108150456-e7962ed7031f diff --git a/api/go.sum b/api/go.sum index 1a6975963..cf2475b0f 100644 --- a/api/go.sum +++ b/api/go.sum @@ -62,6 +62,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deydra71/lib-common/modules/common v0.0.0-20240108150456-e7962ed7031f h1:1DOq6SRvQLbPRrwtoZuA3UyQPMLNYqM2VyNX6JYKgmo= +github.com/deydra71/lib-common/modules/common v0.0.0-20240108150456-e7962ed7031f/go.mod h1:ov4lAbniNUsLqZCBp1RTixpqXc8JlzA5B+yTcCkJXQg= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -220,8 +222,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240106101723-5f7aa263457f h1:ifW2n0TkrS1Wa58DK/N+zAKdGH5XQh4Llk/hefsXzN8= -github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240106101723-5f7aa263457f/go.mod h1:ov4lAbniNUsLqZCBp1RTixpqXc8JlzA5B+yTcCkJXQg= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/api/v1beta1/novaapi_types.go b/api/v1beta1/novaapi_types.go index 7a3b639a4..cc0eca20d 100644 --- a/api/v1beta1/novaapi_types.go +++ b/api/v1beta1/novaapi_types.go @@ -19,6 +19,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -72,6 +73,11 @@ type NovaAPITemplate struct { // +kubebuilder:validation:Optional // Override, provides the ability to override the generated manifest of several child resources. Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` } // APIOverrideSpec to override the generated manifest of several child resources. @@ -153,6 +159,11 @@ type NovaAPISpec struct { // reconfigured to trigger refresh of the in memory cell caches of the // service. RegisteredCells map[string]string `json:"registeredCells"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` } // NovaAPIStatus defines the observed state of NovaAPI diff --git a/api/v1beta1/novacell_types.go b/api/v1beta1/novacell_types.go index 1a54a5b7f..95ba25185 100644 --- a/api/v1beta1/novacell_types.go +++ b/api/v1beta1/novacell_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -173,6 +174,11 @@ type NovaCellSpec struct { // +kubebuilder:validation:Required // ServiceAccount - service account name used internally to provide Nova services the default SA name ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` } // NovaCellStatus defines the observed state of NovaCell diff --git a/api/v1beta1/novacompute_types.go b/api/v1beta1/novacompute_types.go index 45efbc15d..501351fe0 100644 --- a/api/v1beta1/novacompute_types.go +++ b/api/v1beta1/novacompute_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -118,6 +119,11 @@ type NovaComputeSpec struct { // +kubebuilder:validation:Enum=ironic.IronicDriver;fake.FakeDriver // ComputeDriver defines which driver to use for controlling virtualization ComputeDriver string `json:"computeDriver"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` } // NovaComputeStatus defines the observed state of NovaCompute @@ -213,6 +219,7 @@ func NewNovaComputeSpec( ServiceUser: novaCell.ServiceUser, ServiceAccount: novaCell.ServiceAccount, ComputeDriver: computeTemplate.ComputeDriver, + TLS: novaCell.TLS, } return novacomputeSpec } diff --git a/api/v1beta1/novaconductor_types.go b/api/v1beta1/novaconductor_types.go index b6b048277..937f6db03 100644 --- a/api/v1beta1/novaconductor_types.go +++ b/api/v1beta1/novaconductor_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -127,6 +128,11 @@ type NovaConductorSpec struct { // +kubebuilder:validation:Required // ServiceAccount - service account name used internally to provide Nova services the default SA name ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` } // NovaConductorStatus defines the observed state of NovaConductor @@ -196,6 +202,7 @@ func NewNovaConductorSpec( KeystoneAuthURL: novaCell.KeystoneAuthURL, ServiceUser: novaCell.ServiceUser, ServiceAccount: novaCell.ServiceAccount, + TLS: novaCell.TLS, } return conductorSpec } diff --git a/api/v1beta1/novametadata_types.go b/api/v1beta1/novametadata_types.go index ac134b488..773379c22 100644 --- a/api/v1beta1/novametadata_types.go +++ b/api/v1beta1/novametadata_types.go @@ -19,6 +19,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -83,6 +84,11 @@ type NovaMetadataTemplate struct { // +kubebuilder:validation:Optional // Override, provides the ability to override the generated manifest of several child resources. Override MetadataOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.SimpleService `json:"tls,omitempty"` } // MetadataOverrideSpec to override the generated manifest of several child resources. @@ -171,6 +177,11 @@ type NovaMetadataSpec struct { // service. // This is empty for the case when nova-metadata runs within the cell. RegisteredCells map[string]string `json:"registeredCells,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.SimpleService `json:"tls,omitempty"` } // NovaMetadataStatus defines the observed state of NovaMetadata @@ -249,6 +260,7 @@ func NewNovaMetadataSpec( ServiceUser: novaCell.ServiceUser, ServiceAccount: novaCell.ServiceAccount, Override: novaCell.MetadataServiceTemplate.Override, + TLS: novaCell.MetadataServiceTemplate.TLS, } return metadataSpec } diff --git a/api/v1beta1/novanovncproxy_types.go b/api/v1beta1/novanovncproxy_types.go index 1a9e872e4..0e3e5bb4c 100644 --- a/api/v1beta1/novanovncproxy_types.go +++ b/api/v1beta1/novanovncproxy_types.go @@ -19,6 +19,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -79,6 +80,11 @@ type NovaNoVNCProxyTemplate struct { // +kubebuilder:validation:Optional // Override, provides the ability to override the generated manifest of several child resources. Override VNCProxyOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.SimpleService `json:"tls,omitempty"` } // VNCProxyOverrideSpec to override the generated manifest of several child resources. @@ -140,6 +146,11 @@ type NovaNoVNCProxySpec struct { // +kubebuilder:validation:Required // ServiceAccount - service account name used internally to provide Nova services the default SA name ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.SimpleService `json:"tls,omitempty"` } // NovaNoVNCProxyStatus defines the observed state of NovaNoVNCProxy @@ -216,6 +227,7 @@ func NewNovaNoVNCProxySpec( ServiceUser: novaCell.ServiceUser, ServiceAccount: novaCell.ServiceAccount, Override: novaCell.NoVNCProxyServiceTemplate.Override, + TLS: novaCell.NoVNCProxyServiceTemplate.TLS, } return noVNCProxSpec } diff --git a/api/v1beta1/novascheduler_types.go b/api/v1beta1/novascheduler_types.go index 27ae38f89..054080f11 100644 --- a/api/v1beta1/novascheduler_types.go +++ b/api/v1beta1/novascheduler_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -127,6 +128,11 @@ type NovaSchedulerSpec struct { // reconfigured to trigger refresh of the in memory cell caches of the // service. RegisteredCells map[string]string `json:"registeredCells"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` } // NovaSchedulerStatus defines the observed state of NovaScheduler diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 2f7916260..2ce3e86d3 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -213,6 +213,7 @@ func (in *NovaAPISpec) DeepCopyInto(out *NovaAPISpec) { (*out)[key] = val } } + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaAPISpec. @@ -298,6 +299,7 @@ func (in *NovaAPITemplate) DeepCopyInto(out *NovaAPITemplate) { copy(*out, *in) } in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaAPITemplate. @@ -405,6 +407,7 @@ func (in *NovaCellSpec) DeepCopyInto(out *NovaCellSpec) { (*out)[key] = *val.DeepCopy() } } + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaCellSpec. @@ -580,6 +583,7 @@ func (in *NovaComputeSpec) DeepCopyInto(out *NovaComputeSpec) { *out = *in out.Debug = in.Debug in.NovaServiceBase.DeepCopyInto(&out.NovaServiceBase) + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaComputeSpec. @@ -755,6 +759,7 @@ func (in *NovaConductorSpec) DeepCopyInto(out *NovaConductorSpec) { *out = *in out.Debug = in.Debug in.NovaServiceBase.DeepCopyInto(&out.NovaServiceBase) + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaConductorSpec. @@ -985,6 +990,7 @@ func (in *NovaMetadataSpec) DeepCopyInto(out *NovaMetadataSpec) { (*out)[key] = val } } + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaMetadataSpec. @@ -1075,6 +1081,7 @@ func (in *NovaMetadataTemplate) DeepCopyInto(out *NovaMetadataTemplate) { copy(*out, *in) } in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaMetadataTemplate. @@ -1167,6 +1174,7 @@ func (in *NovaNoVNCProxySpec) DeepCopyInto(out *NovaNoVNCProxySpec) { out.Debug = in.Debug in.NovaServiceBase.DeepCopyInto(&out.NovaServiceBase) in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaNoVNCProxySpec. @@ -1257,6 +1265,7 @@ func (in *NovaNoVNCProxyTemplate) DeepCopyInto(out *NovaNoVNCProxyTemplate) { copy(*out, *in) } in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaNoVNCProxyTemplate. @@ -1355,6 +1364,7 @@ func (in *NovaSchedulerSpec) DeepCopyInto(out *NovaSchedulerSpec) { (*out)[key] = val } } + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaSchedulerSpec. diff --git a/config/crd/bases/nova.openstack.org_nova.yaml b/config/crd/bases/nova.openstack.org_nova.yaml index 6fefa0194..7fe1fbb3c 100644 --- a/config/crd/bases/nova.openstack.org_nova.yaml +++ b/config/crd/bases/nova.openstack.org_nova.yaml @@ -326,6 +326,36 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object type: object cellTemplates: additionalProperties: @@ -745,6 +775,18 @@ spec: value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object type: object noVNCProxyServiceTemplate: description: NoVNCProxyServiceTemplate - defines the novncproxy @@ -1040,6 +1082,18 @@ spec: value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object type: object nodeSelector: additionalProperties: @@ -1497,6 +1551,17 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object type: object nodeSelector: additionalProperties: diff --git a/config/crd/bases/nova.openstack.org_novaapis.yaml b/config/crd/bases/nova.openstack.org_novaapis.yaml index e520a11b8..ef45bc78b 100644 --- a/config/crd/bases/nova.openstack.org_novaapis.yaml +++ b/config/crd/bases/nova.openstack.org_novaapis.yaml @@ -383,6 +383,36 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - apiDatabaseHostname - cell0DatabaseHostname diff --git a/config/crd/bases/nova.openstack.org_novacells.yaml b/config/crd/bases/nova.openstack.org_novacells.yaml index 0170ac22c..6a7bc9b6d 100644 --- a/config/crd/bases/nova.openstack.org_novacells.yaml +++ b/config/crd/bases/nova.openstack.org_novacells.yaml @@ -455,6 +455,17 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object type: object noVNCProxyServiceTemplate: description: NoVNCProxyServiceTemplate - defines the novvncproxy service @@ -729,6 +740,17 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object type: object nodeSelector: additionalProperties: @@ -861,6 +883,14 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - cellDatabaseHostname - cellName diff --git a/config/crd/bases/nova.openstack.org_novacomputes.yaml b/config/crd/bases/nova.openstack.org_novacomputes.yaml index 271191c96..e40199c89 100644 --- a/config/crd/bases/nova.openstack.org_novacomputes.yaml +++ b/config/crd/bases/nova.openstack.org_novacomputes.yaml @@ -189,6 +189,14 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - cellName - computeDriver diff --git a/config/crd/bases/nova.openstack.org_novaconductors.yaml b/config/crd/bases/nova.openstack.org_novaconductors.yaml index 1371c5a44..a588a3da4 100644 --- a/config/crd/bases/nova.openstack.org_novaconductors.yaml +++ b/config/crd/bases/nova.openstack.org_novaconductors.yaml @@ -201,6 +201,14 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - cellName - keystoneAuthURL diff --git a/config/crd/bases/nova.openstack.org_novametadata.yaml b/config/crd/bases/nova.openstack.org_novametadata.yaml index d18df7ebb..77cc99e42 100644 --- a/config/crd/bases/nova.openstack.org_novametadata.yaml +++ b/config/crd/bases/nova.openstack.org_novametadata.yaml @@ -378,6 +378,17 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object required: - keystoneAuthURL - secret diff --git a/config/crd/bases/nova.openstack.org_novanovncproxies.yaml b/config/crd/bases/nova.openstack.org_novanovncproxies.yaml index fb77c4c9b..3073a61ae 100644 --- a/config/crd/bases/nova.openstack.org_novanovncproxies.yaml +++ b/config/crd/bases/nova.openstack.org_novanovncproxies.yaml @@ -356,6 +356,17 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object required: - cellDatabaseHostname - cellName diff --git a/config/crd/bases/nova.openstack.org_novaschedulers.yaml b/config/crd/bases/nova.openstack.org_novaschedulers.yaml index 4fe449479..d169f8993 100644 --- a/config/crd/bases/nova.openstack.org_novaschedulers.yaml +++ b/config/crd/bases/nova.openstack.org_novaschedulers.yaml @@ -204,6 +204,14 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - apiDatabaseHostname - cell0DatabaseHostname diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index 2a2ddc1a2..c6d720893 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -845,6 +845,7 @@ func (r *NovaReconciler) ensureCell( ServiceUser: instance.Spec.ServiceUser, KeystoneAuthURL: keystoneAuthURL, ServiceAccount: instance.RbacResourceName(), + TLS: instance.Spec.APIServiceTemplate.TLS.Ca, } if cellTemplate.HasAPIAccess { cellSpec.APIDatabaseHostname = apiDB.GetDatabaseHostname() @@ -1005,6 +1006,7 @@ func (r *NovaReconciler) ensureAPI( ServiceUser: instance.Spec.ServiceUser, ServiceAccount: instance.RbacResourceName(), RegisteredCells: instance.Status.RegisteredCells, + TLS: instance.Spec.APIServiceTemplate.TLS, } api := &novav1.NovaAPI{ ObjectMeta: metav1.ObjectMeta{ @@ -1080,6 +1082,7 @@ func (r *NovaReconciler) ensureScheduler( ServiceUser: instance.Spec.ServiceUser, ServiceAccount: instance.RbacResourceName(), RegisteredCells: instance.Status.RegisteredCells, + TLS: instance.Spec.APIServiceTemplate.TLS.Ca, } scheduler := &novav1.NovaScheduler{ ObjectMeta: metav1.ObjectMeta{ @@ -1427,6 +1430,7 @@ func (r *NovaReconciler) ensureMetadata( KeystoneAuthURL: keystoneAuthURL, ServiceAccount: instance.RbacResourceName(), RegisteredCells: instance.Status.RegisteredCells, + TLS: instance.Spec.MetadataServiceTemplate.TLS, } metadata = &novav1.NovaMetadata{ ObjectMeta: metav1.ObjectMeta{ diff --git a/controllers/novaapi_controller.go b/controllers/novaapi_controller.go index 204655c1e..97616a888 100644 --- a/controllers/novaapi_controller.go +++ b/controllers/novaapi_controller.go @@ -22,11 +22,17 @@ import ( v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" @@ -40,6 +46,7 @@ import ( nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" @@ -201,6 +208,54 @@ func (r *NovaAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re } hashes["cells"] = env.SetValue(cellHash) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + h.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + hashes[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate API service certs secrets + certsHash, ctrlResult, err := instance.Spec.TLS.API.ValidateCertSecrets(ctx, h, instance.Namespace) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + hashes[tls.TLSHashName] = env.SetValue(certsHash) + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + inputHash, err := util.HashOfInputHashes(hashes) if err != nil { return ctrl.Result{}, err @@ -302,6 +357,11 @@ func (r *NovaAPIReconciler) initConditions( condition.InitReason, condition.NetworkAttachmentsReadyInitMessage, ), + condition.UnknownCondition( + condition.TLSInputReadyCondition, + condition.InitReason, + condition.InputReadyInitMessage, + ), ) instance.Status.Conditions.Init(&cl) @@ -357,7 +417,23 @@ func (r *NovaAPIReconciler) generateConfigs( "default_user_domain": "Default", // fixme "transport_url": string(secret.Data[TransportURLSelector]), "log_file": "/var/log/nova/nova-api.log", + "tls": false, + } + // create httpd vhost template parameters + httpdVhostConfig := map[string]interface{}{} + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + endptConfig := map[string]interface{}{} + endptConfig["ServerName"] = fmt.Sprintf("nova-%s.%s.svc", endpt.String(), instance.Namespace) + endptConfig["tls"] = false // default TLS to false, and set it bellow to true if enabled + if instance.Spec.TLS.API.Enabled(endpt) { + templateParameters["tls"] = true + endptConfig["tls"] = true + endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + } + httpdVhostConfig[endpt.String()] = endptConfig } + templateParameters["VHosts"] = httpdVhostConfig extraData := map[string]string{} if instance.Spec.CustomServiceConfig != "" { extraData["02-nova-override.conf"] = instance.Spec.CustomServiceConfig @@ -386,7 +462,18 @@ func (r *NovaAPIReconciler) ensureDeployment( ) (ctrl.Result, error) { Log := r.GetLogger(ctx) - ss := statefulset.NewStatefulSet(novaapi.StatefulSet(instance, inputHash, getAPIServiceLabels(), annotations), r.RequeueTimeout) + ssSpec, err := novaapi.StatefulSet(instance, inputHash, getAPIServiceLabels(), annotations) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + ss := statefulset.NewStatefulSet(ssSpec, r.RequeueTimeout) ctrlResult, err := ss.CreateOrPatch(ctx, h) if err != nil && !k8s_errors.IsNotFound(err) { Log.Error(err, "Deployment failed") @@ -552,7 +639,12 @@ func (r *NovaAPIReconciler) ensureServiceExposed( } // create service - end - // TODO: TLS, pass in https as protocol + // if TLS is enabled + if instance.Spec.TLS.API.Enabled(endpointType) { + // set endpoint protocol to https + data.Protocol = ptr.To(service.ProtocolHTTPS) + } + apiEndpoints[string(endpointType)], err = svc.GetAPIEndpoint( svcOverride.EndpointURL, data.Protocol, data.Path) if err != nil { @@ -663,8 +755,92 @@ func getAPIServiceLabels() map[string]string { } } +func (r *NovaAPIReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("NovaAPI") + + for _, field := range apiWatchFields { + crList := &novav1.NovaAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +// fields to index to reconcile when change +const ( + caBundleSecretNameField = ".spec.tls.caBundleSecretName" + tlsAPIInternalField = ".spec.tls.api.internal.secretName" + tlsAPIPublicField = ".spec.tls.api.public.secretName" +) + +var ( + apiWatchFields = []string{ + caBundleSecretNameField, + tlsAPIInternalField, + tlsAPIPublicField, + } +) + // SetupWithManager sets up the controller with the Manager. func (r *NovaAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaAPI{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaAPI) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsAPIInternalField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaAPI{}, tlsAPIInternalField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaAPI) + if cr.Spec.TLS.API.Internal.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Internal.SecretName} + }); err != nil { + return err + } + + // index tlsAPIPublicField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaAPI{}, tlsAPIPublicField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaAPI) + if cr.Spec.TLS.API.Public.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Public.SecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&novav1.NovaAPI{}). Owns(&v1.StatefulSet{}). @@ -673,5 +849,10 @@ func (r *NovaAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(r.GetSecretMapperFor(&novav1.NovaAPIList{}, context.TODO()))). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } diff --git a/controllers/novacompute_controller.go b/controllers/novacompute_controller.go index fe171f9c1..8bc980612 100644 --- a/controllers/novacompute_controller.go +++ b/controllers/novacompute_controller.go @@ -22,10 +22,15 @@ import ( v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" @@ -36,6 +41,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" @@ -161,6 +167,39 @@ func (r *NovaComputeReconciler) Reconcile(ctx context.Context, req ctrl.Request) // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + h.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + hashes[tls.CABundleKey] = env.SetValue(hash) + } + } + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + err = r.ensureConfigs(ctx, h, instance, &hashes, secret) if err != nil { return ctrl.Result{}, err @@ -236,6 +275,11 @@ func (r *NovaComputeReconciler) initConditions( condition.InitReason, condition.NetworkAttachmentsReadyInitMessage, ), + condition.UnknownCondition( + condition.TLSInputReadyCondition, + condition.InitReason, + condition.InputReadyInitMessage, + ), ) instance.Status.Conditions.Init(&cl) @@ -393,13 +437,70 @@ func getComputeServiceLabels(cell string) map[string]string { } } +func (r *NovaComputeReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("NovaCompute") + + for _, field := range cmpWatchFields { + crList := &novav1.NovaComputeList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +// fields to index to reconcile when change +var ( + cmpWatchFields = []string{ + caBundleSecretNameField, + } +) + // SetupWithManager sets up the controller with the Manager. func (r *NovaComputeReconciler) SetupWithManager(mgr ctrl.Manager) error { + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaCompute{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaCompute) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&novav1.NovaCompute{}). Owns(&v1.StatefulSet{}). Owns(&corev1.Secret{}). Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(r.GetSecretMapperFor(&novav1.NovaComputeList{}, context.TODO()))). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } diff --git a/controllers/novaconductor_controller.go b/controllers/novaconductor_controller.go index 498ceddce..b6c08212d 100644 --- a/controllers/novaconductor_controller.go +++ b/controllers/novaconductor_controller.go @@ -24,10 +24,15 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" @@ -39,6 +44,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" "github.com/openstack-k8s-operators/nova-operator/pkg/novaconductor" @@ -163,6 +169,39 @@ func (r *NovaConductorReconciler) Reconcile(ctx context.Context, req ctrl.Reques // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + h.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + hashes[tls.CABundleKey] = env.SetValue(hash) + } + } + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + err = r.ensureConfigs(ctx, h, instance, &hashes, secret) if err != nil { return ctrl.Result{}, err @@ -257,6 +296,11 @@ func (r *NovaConductorReconciler) initConditions( condition.InitReason, condition.NetworkAttachmentsReadyInitMessage, ), + condition.UnknownCondition( + condition.TLSInputReadyCondition, + condition.InitReason, + condition.InputReadyInitMessage, + ), ) instance.Status.Conditions.Init(&cl) @@ -469,8 +513,60 @@ func (r *NovaConductorReconciler) cleanServiceFromNovaDb( return cleanNovaServiceFromNovaDb(ctx, computeClient, "nova-conductor") } +func (r *NovaConductorReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("NovaConductor") + + for _, field := range cdWatchFields { + crList := &novav1.NovaConductorList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +// fields to index to reconcile when change +var ( + cdWatchFields = []string{ + caBundleSecretNameField, + } +) + // SetupWithManager sets up the controller with the Manager. func (r *NovaConductorReconciler) SetupWithManager(mgr ctrl.Manager) error { + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaConductor{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaConductor) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&novav1.NovaConductor{}). Owns(&v1.StatefulSet{}). @@ -478,5 +574,10 @@ func (r *NovaConductorReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(r.GetSecretMapperFor(&novav1.NovaConductorList{}, context.TODO()))). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } diff --git a/controllers/novametadata_controller.go b/controllers/novametadata_controller.go index e3deafd39..58b8c3943 100644 --- a/controllers/novametadata_controller.go +++ b/controllers/novametadata_controller.go @@ -23,11 +23,16 @@ import ( v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" @@ -40,6 +45,7 @@ import ( common_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" "github.com/openstack-k8s-operators/nova-operator/pkg/nova" @@ -167,6 +173,55 @@ func (r *NovaMetadataReconciler) Reconcile(ctx context.Context, req ctrl.Request // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + h.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + hashes[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate metadata service cert secret + if instance.Spec.TLS.Enabled() { + hash, ctrlResult, err := instance.Spec.TLS.ValidateCertSecret(ctx, h, instance.Namespace) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + hashes[tls.TLSHashName] = env.SetValue(hash) + } + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + err = r.ensureConfigs(ctx, h, instance, &hashes, secret) if err != nil { return ctrl.Result{}, err @@ -283,6 +338,11 @@ func (r *NovaMetadataReconciler) initConditions( condition.InitReason, novav1.NovaComputeServiceConfigInitMessage, ), + condition.UnknownCondition( + condition.TLSInputReadyCondition, + condition.InitReason, + condition.InputReadyInitMessage, + ), ) instance.Status.Conditions.Init(&cl) @@ -331,6 +391,8 @@ func (r *NovaMetadataReconciler) generateConfigs( "metadata_secret": string(secret.Data[MetadataSecretSelector]), "log_file": "/var/log/nova/nova-metadata.log", "transport_url": string(secret.Data[TransportURLSelector]), + "tls": false, + "ServerName": fmt.Sprintf("%s.%s.svc", novametadata.ServiceName, instance.Namespace), } if instance.Spec.CellName == "" { @@ -344,6 +406,13 @@ func (r *NovaMetadataReconciler) generateConfigs( templateParameters["local_metadata_per_cell"] = true } + // create httpd tls template parameters + if instance.Spec.TLS.GenericService.Enabled() { + templateParameters["tls"] = true + templateParameters["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", novametadata.ServiceName) + templateParameters["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", novametadata.ServiceName) + } + extraData := map[string]string{} if instance.Spec.CustomServiceConfig != "" { extraData["02-nova-override.conf"] = instance.Spec.CustomServiceConfig @@ -373,7 +442,18 @@ func (r *NovaMetadataReconciler) ensureDeployment( Log := r.GetLogger(ctx) serviceLabels := getMetadataServiceLabels(instance.Spec.CellName) - ss := statefulset.NewStatefulSet(novametadata.StatefulSet(instance, inputHash, serviceLabels, annotations), r.RequeueTimeout) + ssSpec, err := novametadata.StatefulSet(instance, inputHash, serviceLabels, annotations) + if err != nil { + Log.Error(err, "Deployment failed") + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + ss := statefulset.NewStatefulSet(ssSpec, r.RequeueTimeout) ctrlResult, err := ss.CreateOrPatch(ctx, h) if err != nil && !k8s_errors.IsNotFound(err) { Log.Error(err, "Deployment failed") @@ -517,9 +597,14 @@ func (r *NovaMetadataReconciler) ensureServiceExposed( } // create service - end - // TODO: TLS, pass in https as protocol + // if TLS is enabled + proto := ptr.To(service.ProtocolHTTP) + if instance.Spec.TLS.Enabled() { + // set endpoint protocol to https + proto = ptr.To(service.ProtocolHTTPS) + } apiEndpoint, err := svc.GetAPIEndpoint( - nil, ptr.To(service.ProtocolHTTP), "") + nil, proto, "") if err != nil { return "", ctrl.Result{}, err } @@ -632,8 +717,77 @@ func (r *NovaMetadataReconciler) generateNeutronConfigs( return nil } +func (r *NovaMetadataReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("NovaMetadata") + + for _, field := range metaWatchFields { + crList := &novav1.NovaMetadataList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +// fields to index to reconcile when change +const ( + tlsMetadataField = ".spec.tls.secretName" +) + +var ( + metaWatchFields = []string{ + caBundleSecretNameField, + tlsMetadataField, + } +) + // SetupWithManager sets up the controller with the Manager. func (r *NovaMetadataReconciler) SetupWithManager(mgr ctrl.Manager) error { + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaMetadata{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaMetadata) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsMetadataField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaMetadata{}, tlsMetadataField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaMetadata) + if cr.Spec.TLS.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.SecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&novav1.NovaMetadata{}). Owns(&v1.StatefulSet{}). @@ -641,5 +795,10 @@ func (r *NovaMetadataReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(r.GetSecretMapperFor(&novav1.NovaMetadataList{}, context.TODO()))). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } diff --git a/controllers/novanovncproxy_controller.go b/controllers/novanovncproxy_controller.go index 90e947924..535315770 100644 --- a/controllers/novanovncproxy_controller.go +++ b/controllers/novanovncproxy_controller.go @@ -22,10 +22,15 @@ import ( v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" @@ -37,6 +42,7 @@ import ( nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" "github.com/openstack-k8s-operators/nova-operator/pkg/nova" @@ -161,6 +167,56 @@ func (r *NovaNoVNCProxyReconciler) Reconcile(ctx context.Context, req ctrl.Reque // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + h.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + hashes[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate metadata service cert secret + if instance.Spec.TLS.Enabled() { + hash, ctrlResult, err := instance.Spec.TLS.ValidateCertSecret(ctx, h, instance.Namespace) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + hashes[tls.TLSHashName] = env.SetValue(hash) + } + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + result, err = r.ensureServiceExposed(ctx, h, instance) if (err != nil || result != ctrl.Result{}) { // We can ignore RequeueAfter as we are watching the Service resource @@ -269,6 +325,11 @@ func (r *NovaNoVNCProxyReconciler) initConditions( condition.InitReason, condition.NetworkAttachmentsReadyInitMessage, ), + condition.UnknownCondition( + condition.TLSInputReadyCondition, + condition.InitReason, + condition.InputReadyInitMessage, + ), ) instance.Status.Conditions.Init(&cl) @@ -292,11 +353,14 @@ func (r *NovaNoVNCProxyReconciler) generateConfigs( "cell_db_address": instance.Spec.CellDatabaseHostname, "cell_db_port": 3306, "transport_url": string(secret.Data[TransportURLSelector]), - "openstack_cacert": "", // fixme "openstack_region_name": "regionOne", // fixme "default_project_domain": "Default", // fixme "default_user_domain": "Default", // fixme } + if instance.Spec.TLS.GenericService.Enabled() { + templateParameters["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", novncproxy.ServiceName) + templateParameters["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", novncproxy.ServiceName) + } extraData := map[string]string{} if instance.Spec.CustomServiceConfig != "" { extraData["02-nova-override.conf"] = instance.Spec.CustomServiceConfig @@ -326,8 +390,18 @@ func (r *NovaNoVNCProxyReconciler) ensureDeployment( Log := r.GetLogger(ctx) serviceLabels := getNoVNCProxyServiceLabels(instance.Spec.CellName) - ss := statefulset.NewStatefulSet( - novncproxy.StatefulSet(instance, inputHash, serviceLabels, annotations), r.RequeueTimeout) + ssSpec, err := novncproxy.StatefulSet(instance, inputHash, serviceLabels, annotations) + if err != nil { + Log.Info("Deployment failed") + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + ss := statefulset.NewStatefulSet(ssSpec, r.RequeueTimeout) ctrlResult, err := ss.CreateOrPatch(ctx, h) if err != nil && !k8s_errors.IsNotFound(err) { Log.Info("Deployment failed") @@ -506,8 +580,77 @@ func getNoVNCProxyServiceLabels(cell string) map[string]string { } } +func (r *NovaNoVNCProxyReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("NovaNoVNCProxy") + + for _, field := range noVNCProxyWatchFields { + crList := &novav1.NovaNoVNCProxyList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +// fields to index to reconcile when change +const ( + tlsNoVNCProxyField = ".spec.tls.secretName" +) + +var ( + noVNCProxyWatchFields = []string{ + caBundleSecretNameField, + tlsNoVNCProxyField, + } +) + // SetupWithManager sets up the controller with the Manager. func (r *NovaNoVNCProxyReconciler) SetupWithManager(mgr ctrl.Manager) error { + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaNoVNCProxy{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaNoVNCProxy) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsNoVNCProxyField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaNoVNCProxy{}, tlsNoVNCProxyField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaNoVNCProxy) + if cr.Spec.TLS.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.SecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&novav1.NovaNoVNCProxy{}). Owns(&v1.StatefulSet{}). @@ -515,5 +658,10 @@ func (r *NovaNoVNCProxyReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(r.GetSecretMapperFor(&novav1.NovaNoVNCProxyList{}, context.TODO()))). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } diff --git a/controllers/novascheduler_controller.go b/controllers/novascheduler_controller.go index 76a5becd4..6f94a5589 100644 --- a/controllers/novascheduler_controller.go +++ b/controllers/novascheduler_controller.go @@ -23,10 +23,15 @@ import ( v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" @@ -37,6 +42,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" @@ -161,6 +167,39 @@ func (r *NovaSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Reques // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + h.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + hashes[tls.CABundleKey] = env.SetValue(hash) + } + } + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + err = r.ensureConfigs(ctx, h, instance, &hashes, secret) if err != nil { return ctrl.Result{}, err @@ -205,14 +244,71 @@ func (r *NovaSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } +func (r *NovaSchedulerReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("NovaScheduler") + + for _, field := range schedulerWatchFields { + crList := &novav1.NovaSchedulerList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +// fields to index to reconcile when change +var ( + schedulerWatchFields = []string{ + caBundleSecretNameField, + } +) + // SetupWithManager sets up the controller with the Manager. func (r *NovaSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &novav1.NovaScheduler{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*novav1.NovaScheduler) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&novav1.NovaScheduler{}). Owns(&v1.StatefulSet{}). Owns(&corev1.Secret{}). Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(r.GetSecretMapperFor(&novav1.NovaSchedulerList{}, context.TODO()))). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } @@ -265,6 +361,11 @@ func (r *NovaSchedulerReconciler) initConditions( condition.InitReason, condition.NetworkAttachmentsReadyInitMessage, ), + condition.UnknownCondition( + condition.TLSInputReadyCondition, + condition.InitReason, + condition.InputReadyInitMessage, + ), ) instance.Status.Conditions.Init(&cl) diff --git a/go.mod b/go.mod index ed08e7677..c3926d636 100644 --- a/go.mod +++ b/go.mod @@ -88,3 +88,5 @@ replace github.com/openstack-k8s-operators/nova-operator/api => ./api // mschuppert: map to latest commit from release-4.13 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/deydra71/lib-common/modules/common v0.0.0-20240108150456-e7962ed7031f diff --git a/go.sum b/go.sum index d2632a402..9b0645ff9 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deydra71/lib-common/modules/common v0.0.0-20240108150456-e7962ed7031f h1:1DOq6SRvQLbPRrwtoZuA3UyQPMLNYqM2VyNX6JYKgmo= +github.com/deydra71/lib-common/modules/common v0.0.0-20240108150456-e7962ed7031f/go.mod h1:ov4lAbniNUsLqZCBp1RTixpqXc8JlzA5B+yTcCkJXQg= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -238,8 +240,6 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.3.1-0.20240104150635-c github.com/openstack-k8s-operators/infra-operator/apis v0.3.1-0.20240104150635-c4ffc51e0752/go.mod h1:y4qeIT1ubUm9SKrvhVTuEYWSm0so38P5Hu3ZpdMJMek= github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20240104144437-5355d932c316 h1:IwTuIoC78bbp3awd8P0tWeknCe2jNLB1FCJDIwI/2Pg= github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20240104144437-5355d932c316/go.mod h1:qx+z+k0RMK8Vcl5Nug6bOScEg7ROSxEV4FFy0gjcQDQ= -github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240106101723-5f7aa263457f h1:ifW2n0TkrS1Wa58DK/N+zAKdGH5XQh4Llk/hefsXzN8= -github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240106101723-5f7aa263457f/go.mod h1:ov4lAbniNUsLqZCBp1RTixpqXc8JlzA5B+yTcCkJXQg= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20240106101723-5f7aa263457f h1:bXaUzrP+JiBHSCQwmndGjD5nafFAK+t8X2oXVP8hj3c= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20240106101723-5f7aa263457f/go.mod h1:Lb1xXXJpbWxU619nemBD4XQy+rqRhzyNcGji9zZbG20= github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240106101723-5f7aa263457f h1:Tbw6HGO793AyaxhheNV7r+ftunFHzVBJKtgFG/RNLc0= diff --git a/pkg/nova/cellmapping.go b/pkg/nova/cellmapping.go index 0b63ced51..addc768d1 100644 --- a/pkg/nova/cellmapping.go +++ b/pkg/nova/cellmapping.go @@ -40,6 +40,22 @@ func CellMappingJob( jobName := instance.Name + "-" + cell.Spec.CellName + "-cell-mapping" + volumes := []corev1.Volume{ + GetConfigVolume(configName), + GetScriptVolume(scriptName), + } + volumeMounts := []corev1.VolumeMount{ + GetConfigVolumeMount(), + GetScriptVolumeMount(), + GetKollaConfigVolumeMount("cell-mapping"), + } + + // add CA cert if defined + if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.APIServiceTemplate.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.APIServiceTemplate.TLS.CreateVolumeMounts(nil)...) + } + job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: jobName, @@ -51,10 +67,7 @@ func CellMappingJob( Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), - Volumes: []corev1.Volume{ - GetConfigVolume(configName), - GetScriptVolume(scriptName), - }, + Volumes: volumes, Containers: []corev1.Container{ { Name: "nova-manage", @@ -66,12 +79,8 @@ func CellMappingJob( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - GetConfigVolumeMount(), - GetScriptVolumeMount(), - GetKollaConfigVolumeMount("cell-mapping"), - }, + Env: env, + VolumeMounts: volumeMounts, }, }, }, diff --git a/pkg/nova/host_discover.go b/pkg/nova/host_discover.go index c7104cc0e..3690528df 100644 --- a/pkg/nova/host_discover.go +++ b/pkg/nova/host_discover.go @@ -52,6 +52,22 @@ func HostDiscoveryJob( jobName := instance.Name + "-host-discover" + volumes := []corev1.Volume{ + GetConfigVolume(configName), + GetScriptVolume(scriptName), + } + volumeMounts := []corev1.VolumeMount{ + GetConfigVolumeMount(), + GetScriptVolumeMount(), + GetKollaConfigVolumeMount("host-discover"), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: jobName, @@ -63,10 +79,7 @@ func HostDiscoveryJob( Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.Spec.ServiceAccount, - Volumes: []corev1.Volume{ - GetConfigVolume(configName), - GetScriptVolume(scriptName), - }, + Volumes: volumes, Containers: []corev1.Container{ { Name: "nova-manage", @@ -78,12 +91,8 @@ func HostDiscoveryJob( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - GetConfigVolumeMount(), - GetScriptVolumeMount(), - GetKollaConfigVolumeMount("host-discover"), - }, + Env: env, + VolumeMounts: volumeMounts, }, }, }, diff --git a/pkg/novaapi/deployment.go b/pkg/novaapi/deployment.go index b29c709ea..30dca1d89 100644 --- a/pkg/novaapi/deployment.go +++ b/pkg/novaapi/deployment.go @@ -20,6 +20,8 @@ import ( common "github.com/openstack-k8s-operators/lib-common/modules/common" affinity "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" env "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" "github.com/openstack-k8s-operators/nova-operator/pkg/nova" @@ -36,7 +38,7 @@ func StatefulSet( configHash string, labels map[string]string, annotations map[string]string, -) *appsv1.StatefulSet { +) (*appsv1.StatefulSet, error) { // This allows the pod to start up slowly. The pod will only be killed // if it does not succeed a probe in 60 seconds. startupProbe := &corev1.Probe{ @@ -85,6 +87,12 @@ func StatefulSet( startupProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(APIServicePort)}, } + + if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + startupProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } } nodeSelector := map[string]string{} @@ -101,6 +109,42 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), + nova.GetLogVolume(), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetLogVolumeMount(), + nova.GetKollaConfigVolumeMount("nova-api"), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + if instance.Spec.TLS.API.Enabled(endpt) { + var tlsEndptCfg tls.GenericService + switch endpt { + case service.EndpointPublic: + tlsEndptCfg = instance.Spec.TLS.API.Public + case service.EndpointInternal: + tlsEndptCfg = instance.Spec.TLS.API.Internal + } + + svc, err := tlsEndptCfg.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } + } + statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -119,10 +163,7 @@ func StatefulSet( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.Spec.ServiceAccount, - Volumes: []corev1.Volume{ - nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), - nova.GetLogVolume(), - }, + Volumes: volumes, Containers: []corev1.Container{ // the first container in a pod is the default selected // by oc log so define the log stream container first. @@ -160,12 +201,8 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(nova.NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - nova.GetConfigVolumeMount(), - nova.GetLogVolumeMount(), - nova.GetKollaConfigVolumeMount("nova-api"), - }, + Env: env, + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, StartupProbe: startupProbe, ReadinessProbe: readinessProbe, @@ -188,5 +225,5 @@ func StatefulSet( }, } - return statefulset + return statefulset, nil } diff --git a/pkg/novacompute/deployment.go b/pkg/novacompute/deployment.go index abef38d88..92a8e3aae 100644 --- a/pkg/novacompute/deployment.go +++ b/pkg/novacompute/deployment.go @@ -91,6 +91,21 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetKollaConfigVolumeMount("nova-compute"), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -109,9 +124,7 @@ func StatefulSet( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.Spec.ServiceAccount, - Volumes: []corev1.Volume{ - nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), - }, + Volumes: volumes, Containers: []corev1.Container{ { Name: instance.Name + "-compute", @@ -123,11 +136,8 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(nova.NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - nova.GetConfigVolumeMount(), - nova.GetKollaConfigVolumeMount("nova-compute"), - }, + Env: env, + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, ReadinessProbe: readinessProbe, LivenessProbe: livenessProbe, diff --git a/pkg/novaconductor/dbsync.go b/pkg/novaconductor/dbsync.go index 8dc324060..b16d0827a 100644 --- a/pkg/novaconductor/dbsync.go +++ b/pkg/novaconductor/dbsync.go @@ -50,6 +50,23 @@ func CellDBSyncJob( env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), + nova.GetScriptVolume(nova.GetScriptSecretName(instance.Name)), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetScriptVolumeMount(), + nova.GetKollaConfigVolumeMount("nova-conductor-dbsync"), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name + "-db-sync", @@ -62,10 +79,7 @@ func CellDBSyncJob( Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.Spec.ServiceAccount, - Volumes: []corev1.Volume{ - nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), - nova.GetScriptVolume(nova.GetScriptSecretName(instance.Name)), - }, + Volumes: volumes, Containers: []corev1.Container{ { Name: instance.Name + "-db-sync", @@ -77,12 +91,8 @@ func CellDBSyncJob( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(nova.NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - nova.GetConfigVolumeMount(), - nova.GetScriptVolumeMount(), - nova.GetKollaConfigVolumeMount("nova-conductor-dbsync"), - }, + Env: env, + VolumeMounts: volumeMounts, }, }, }, diff --git a/pkg/novaconductor/deployment.go b/pkg/novaconductor/deployment.go index 91233bfba..c9d8effe5 100644 --- a/pkg/novaconductor/deployment.go +++ b/pkg/novaconductor/deployment.go @@ -97,6 +97,21 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetKollaConfigVolumeMount("nova-conductor"), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -115,9 +130,7 @@ func StatefulSet( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.Spec.ServiceAccount, - Volumes: []corev1.Volume{ - nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), - }, + Volumes: volumes, Containers: []corev1.Container{ { Name: instance.Name + "-conductor", @@ -129,11 +142,8 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(nova.NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - nova.GetConfigVolumeMount(), - nova.GetKollaConfigVolumeMount("nova-conductor"), - }, + Env: env, + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, ReadinessProbe: readinessProbe, LivenessProbe: livenessProbe, diff --git a/pkg/novametadata/deployment.go b/pkg/novametadata/deployment.go index f7cf36a73..288d9bbc1 100644 --- a/pkg/novametadata/deployment.go +++ b/pkg/novametadata/deployment.go @@ -36,7 +36,7 @@ func StatefulSet( configHash string, labels map[string]string, annotations map[string]string, -) *appsv1.StatefulSet { +) (*appsv1.StatefulSet, error) { // This allows the pod to start up slowly. The pod will only be killed // if it does not succeed a probe in 60 seconds. startupProbe := &corev1.Probe{ @@ -85,6 +85,12 @@ func StatefulSet( startupProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(APIServicePort)}, } + + if instance.Spec.TLS.GenericService.Enabled() { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + startupProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } } nodeSelector := map[string]string{} @@ -101,6 +107,32 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), + nova.GetLogVolume(), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetLogVolumeMount(), + nova.GetKollaConfigVolumeMount("nova-metadata"), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + if instance.Spec.TLS.GenericService.Enabled() { + svc, err := instance.Spec.TLS.GenericService.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(ServiceName)) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(ServiceName)...) + } + statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -119,10 +151,7 @@ func StatefulSet( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.Spec.ServiceAccount, - Volumes: []corev1.Volume{ - nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), - nova.GetLogVolume(), - }, + Volumes: volumes, Containers: []corev1.Container{ // the first container in a pod is the default selected // by oc log so define the log stream container first. @@ -143,10 +172,8 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(nova.NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - nova.GetLogVolumeMount(), - }, + Env: env, + VolumeMounts: []corev1.VolumeMount{nova.GetLogVolumeMount()}, Resources: instance.Spec.Resources, StartupProbe: startupProbe, ReadinessProbe: readinessProbe, @@ -162,12 +189,8 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(nova.NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - nova.GetConfigVolumeMount(), - nova.GetLogVolumeMount(), - nova.GetKollaConfigVolumeMount("nova-metadata"), - }, + Env: env, + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, StartupProbe: startupProbe, ReadinessProbe: readinessProbe, @@ -190,5 +213,5 @@ func StatefulSet( }, } - return statefulset + return statefulset, nil } diff --git a/pkg/novascheduler/deployment.go b/pkg/novascheduler/deployment.go index 9724e8a1e..350cc1652 100644 --- a/pkg/novascheduler/deployment.go +++ b/pkg/novascheduler/deployment.go @@ -106,6 +106,21 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetKollaConfigVolumeMount("nova-scheduler"), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -124,9 +139,7 @@ func StatefulSet( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.Spec.ServiceAccount, - Volumes: []corev1.Volume{ - nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), - }, + Volumes: volumes, Containers: []corev1.Container{ { Name: instance.Name + "-scheduler", @@ -138,11 +151,8 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(nova.NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - nova.GetConfigVolumeMount(), - nova.GetKollaConfigVolumeMount("nova-scheduler"), - }, + Env: env, + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, StartupProbe: startupProbe, ReadinessProbe: readinessProbe, diff --git a/pkg/novncproxy/deployment.go b/pkg/novncproxy/deployment.go index e9f89646a..53f57ef6e 100644 --- a/pkg/novncproxy/deployment.go +++ b/pkg/novncproxy/deployment.go @@ -36,7 +36,7 @@ func StatefulSet( configHash string, labels map[string]string, annotations map[string]string, -) *appsv1.StatefulSet { +) (*appsv1.StatefulSet, error) { // This allows the pod to start up slowly. The pod will only be killed // if it does not succeed a probe in 60 seconds. startupProbe := &corev1.Probe{ @@ -88,6 +88,12 @@ func StatefulSet( Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(NoVNCProxyPort)}, Path: "/vnc_lite.html", } + + if instance.Spec.TLS.GenericService.Enabled() { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + startupProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } } nodeSelector := map[string]string{} @@ -104,6 +110,30 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetKollaConfigVolumeMount("nova-novncproxy"), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + if instance.Spec.TLS.GenericService.Enabled() { + svc, err := instance.Spec.TLS.GenericService.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(ServiceName)) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(ServiceName)...) + } + statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -122,9 +152,7 @@ func StatefulSet( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.Spec.ServiceAccount, - Volumes: []corev1.Volume{ - nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), - }, + Volumes: volumes, Containers: []corev1.Container{ { Name: instance.Name + "-novncproxy", @@ -136,11 +164,8 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(nova.NovaUserID), }, - Env: env, - VolumeMounts: []corev1.VolumeMount{ - nova.GetConfigVolumeMount(), - nova.GetKollaConfigVolumeMount("nova-novncproxy"), - }, + Env: env, + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, StartupProbe: startupProbe, ReadinessProbe: readinessProbe, @@ -163,5 +188,5 @@ func StatefulSet( }, } - return statefulset + return statefulset, nil } diff --git a/templates/nova.conf b/templates/nova.conf index b9c91b16f..d89a8ee3a 100644 --- a/templates/nova.conf +++ b/templates/nova.conf @@ -42,6 +42,14 @@ metadata_workers=1 enabled_apis=metadata {{end}} +{{if eq .service_name "nova-novncproxy"}} +{{ if (index . "SSLCertificateFile") }} +ssl_only=true +cert={{.SSLCertificateFile}} +key={{.SSLCertificateKeyFile}} +{{end}} +{{end}} + [oslo_concurrency] lock_path = /var/lib/nova/tmp @@ -211,7 +219,6 @@ user_domain_name = {{ .default_user_domain}} project_name = service username = {{ .nova_keystone_user }} password = {{ .nova_keystone_password }} -cafile = {{ .openstack_cacert }} region_name = {{ .openstack_region_name }} # This is part of hardening related to CVE-2023-2088 # https://docs.openstack.org/nova/latest/configuration/config.html#keystone_authtoken.service_token_roles_required @@ -226,7 +233,6 @@ user_domain_name = {{ .default_user_domain}} project_name = service username = {{ .nova_keystone_user }} password = {{ .nova_keystone_password }} -cafile = {{ .openstack_cacert }} region_name = {{ .openstack_region_name }} valid_interfaces = internal @@ -238,7 +244,6 @@ user_domain_name = {{ .default_user_domain}} project_name = service username = {{ .nova_keystone_user }} password = {{ .nova_keystone_password }} -cafile = {{ .openstack_cacert }} region_name = {{ .openstack_region_name }} valid_interfaces = internal {{if (index . "debug") }}debug=true{{end}} @@ -251,7 +256,6 @@ user_domain_name = {{ .default_user_domain}} project_name = service username = {{ .nova_keystone_user }} password = {{ .nova_keystone_password }} -cafile = {{ .openstack_cacert }} region_name = {{ .openstack_region_name }} valid_interfaces = internal {{if eq .service_name "nova-metadata"}} @@ -267,7 +271,6 @@ user_domain_name = {{ .default_user_domain}} project_name = service username = {{ .nova_keystone_user }} password = {{ .nova_keystone_password }} -cafile = {{ .openstack_cacert }} region_name = {{ .openstack_region_name }} catalog_info = volumev3:cinderv3:internalURL @@ -279,7 +282,6 @@ user_domain_name = {{ .default_user_domain}} project_name = service username = {{ .nova_keystone_user }} password = {{ .nova_keystone_password }} -cafile = {{ .openstack_cacert }} region_name = {{ .openstack_region_name }} barbican_endpoint_type = internal @@ -292,7 +294,6 @@ user_domain_name = {{ .default_user_domain}} project_name = service username = {{ .nova_keystone_user }} password = {{ .nova_keystone_password }} -cafile = {{ .openstack_cacert }} {{ if (index . "compute_driver") }} {{if eq .compute_driver "ironic.IronicDriver"}} diff --git a/templates/novaapi/config/httpd.conf b/templates/novaapi/config/httpd.conf index 17e35370f..f79ff2ca1 100644 --- a/templates/novaapi/config/httpd.conf +++ b/templates/novaapi/config/httpd.conf @@ -13,6 +13,10 @@ Listen 8774 TypesConfig /etc/mime.types Include conf.modules.d/*.conf +{{- if .tls }} +## TODO: fix default ssl.conf to comment not available tls certs. Than we can remove this condition +Include conf.d/*.conf +{{- end }} LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy @@ -25,32 +29,60 @@ CustomLog /dev/stdout proxy env=forwarded ## set default apache log level to info from warning LogLevel info +{{ range $endpt, $vhost := .VHosts }} +# {{ $endpt }} vhost {{ $vhost.ServerName }} configuration = 2.4> ErrorLogFormat "%M" SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + + ServerName {{ $vhost.ServerName }} + + ## Vhost docroot + DocumentRoot "/var/www/cgi-bin" + + ## Directories, there should at least be a declaration for /var/www/cgi-bin + + Options -Indexes +FollowSymLinks +MultiViews + AllowOverride None + Require all granted + + + ## Logging ErrorLog /dev/stdout + ServerSignature Off CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded ## set nova vhost log level to debug LogLevel debug +{{- if $vhost.tls }} + SetEnvIf X-Forwarded-Proto https HTTPS=1 + + ## SSL directives + SSLEngine on + SSLCertificateFile "{{ $vhost.SSLCertificateFile }}" + SSLCertificateKeyFile "{{ $vhost.SSLCertificateKeyFile }}" +{{- end }} + ## WSGI configuration - WSGIProcessGroup nova-api + WSGIProcessGroup {{ $endpt }} + #WSGIProcessGroup nova-api WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On ## In general we want nova-api to scale via k8s replicas but we need ## two processes per replica to always has a room for a healthecheck query - WSGIDaemonProcess nova-api processes=2 threads=1 user=nova group=nova display-name=nova-api - WSGIScriptAlias / /usr/bin/nova-api-wsgi + WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} processes=2 threads=1 user=nova group=nova + WSGIScriptAlias / "/usr/bin/nova-api-wsgi" +{{ end }} Alias /nova-api /usr/bin/nova-api-wsgi SetHandler wsgi-script Options +ExecCGI - WSGIProcessGroup nova-api + WSGIProcessGroup public WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On diff --git a/templates/novaapi/config/nova-api-config.json b/templates/novaapi/config/nova-api-config.json index 5d4a9edde..388e3c43d 100644 --- a/templates/novaapi/config/nova-api-config.json +++ b/templates/novaapi/config/nova-api-config.json @@ -25,6 +25,28 @@ "dest": "/etc/httpd/conf/httpd.conf", "owner": "apache", "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "nova", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "nova", + "perm": "0600", + "optional": true, + "merge": true } ], "permissions": [ diff --git a/templates/novaapi/config/ssl.conf b/templates/novaapi/config/ssl.conf new file mode 100644 index 000000000..e3da4ecb2 --- /dev/null +++ b/templates/novaapi/config/ssl.conf @@ -0,0 +1,21 @@ + + SSLRandomSeed startup builtin + SSLRandomSeed startup file:/dev/urandom 512 + SSLRandomSeed connect builtin + SSLRandomSeed connect file:/dev/urandom 512 + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + SSLPassPhraseDialog builtin + SSLSessionCache "shmcb:/var/cache/mod_ssl/scache(512000)" + SSLSessionCacheTimeout 300 + Mutex default + SSLCryptoDevice builtin + SSLHonorCipherOrder On + SSLUseStapling Off + SSLStaplingCache "shmcb:/run/httpd/ssl_stapling(32768)" + SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4:!3DES + SSLProtocol all -SSLv2 -SSLv3 -TLSv1 + SSLOptions StdEnvVars + diff --git a/templates/novametadata/config/httpd.conf b/templates/novametadata/config/httpd.conf index b66a707c0..14bfb5fd5 100644 --- a/templates/novametadata/config/httpd.conf +++ b/templates/novametadata/config/httpd.conf @@ -13,6 +13,10 @@ Listen 8775 TypesConfig /etc/mime.types Include conf.modules.d/*.conf +{{- if .tls }} +## TODO: fix default ssl.conf to comment not available tls certs. Than we can remove this condition +Include conf.d/*.conf +{{- end }} LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy @@ -30,12 +34,24 @@ LogLevel info ErrorLogFormat "%M" SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + + ServerName {{ .ServerName }} + ErrorLog /dev/stdout CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded ## set nova vhost log level to debug LogLevel debug +{{- if .tls }} + SetEnvIf X-Forwarded-Proto https HTTPS=1 + + ## SSL directives + SSLEngine on + SSLCertificateFile "{{ .SSLCertificateFile }}" + SSLCertificateKeyFile "{{ .SSLCertificateKeyFile }}" +{{- end }} + ## WSGI configuration WSGIProcessGroup nova-metadata WSGIApplicationGroup %{GLOBAL} diff --git a/templates/novametadata/config/nova-metadata-config.json b/templates/novametadata/config/nova-metadata-config.json index d005385e6..f4af32351 100644 --- a/templates/novametadata/config/nova-metadata-config.json +++ b/templates/novametadata/config/nova-metadata-config.json @@ -6,25 +6,47 @@ "dest": "/etc/nova/nova.conf", "owner": "nova", "perm": "0600" - }, - { + }, + { "source": "/var/lib/openstack/config/01-nova.conf", "dest": "/etc/nova/nova.conf.d/01-nova.conf", "owner": "nova", "perm": "0600" - }, - { + }, + { "source": "/var/lib/openstack/config/02-nova-override.conf", "dest": "/etc/nova/nova.conf.d/02-nova-override.conf", "owner": "nova", "perm": "0600", "optional": true - }, + }, { "source": "/var/lib/openstack/config/httpd.conf", "dest": "/etc/httpd/conf/httpd.conf", "owner": "apache", "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "nova", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "nova", + "perm": "0600", + "optional": true, + "merge": true } ], "permissions": [ diff --git a/templates/novametadata/config/ssl.conf b/templates/novametadata/config/ssl.conf new file mode 100644 index 000000000..e3da4ecb2 --- /dev/null +++ b/templates/novametadata/config/ssl.conf @@ -0,0 +1,21 @@ + + SSLRandomSeed startup builtin + SSLRandomSeed startup file:/dev/urandom 512 + SSLRandomSeed connect builtin + SSLRandomSeed connect file:/dev/urandom 512 + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + SSLPassPhraseDialog builtin + SSLSessionCache "shmcb:/var/cache/mod_ssl/scache(512000)" + SSLSessionCacheTimeout 300 + Mutex default + SSLCryptoDevice builtin + SSLHonorCipherOrder On + SSLUseStapling Off + SSLStaplingCache "shmcb:/run/httpd/ssl_stapling(32768)" + SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4:!3DES + SSLProtocol all -SSLv2 -SSLv3 -TLSv1 + SSLOptions StdEnvVars + diff --git a/templates/novanovncproxy/config/nova-novncproxy-config.json b/templates/novanovncproxy/config/nova-novncproxy-config.json index e3f5510b0..10809db01 100644 --- a/templates/novanovncproxy/config/nova-novncproxy-config.json +++ b/templates/novanovncproxy/config/nova-novncproxy-config.json @@ -19,6 +19,22 @@ "owner": "nova", "perm": "0600", "optional": true + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "nova", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "nova", + "perm": "0600", + "optional": true, + "merge": true } ] } diff --git a/test/functional/base_test.go b/test/functional/base_test.go index cfe1f807b..741f8683b 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -43,6 +43,9 @@ const ( // asserts using `Consistently`. consistencyTimeout = timeout ironicComputeName = "ironic-compute" + //PublicCertSecretName = "public-tls-certs" + //InternalCertSecretName = "internal-tls-certs" + //CABundleSecretName = "combined-ca-bundle" ) type NovaAPIFixture struct { @@ -657,6 +660,9 @@ type NovaNames struct { APIKeystoneEndpointName types.NamespacedName APIStatefulSetName types.NamespacedName APIConfigDataName types.NamespacedName + InternalCertSecretName types.NamespacedName + PublicCertSecretName types.NamespacedName + CaBundleSecretName types.NamespacedName // refers internal API network for all Nova services (not just nova API) InternalAPINetworkNADName types.NamespacedName SchedulerName types.NamespacedName @@ -727,6 +733,16 @@ func GetNovaNames(novaName types.NamespacedName, cellNames []string) NovaNames { Namespace: novaAPI.Namespace, Name: novaAPI.Name + "-config-data", }, + InternalCertSecretName: types.NamespacedName{ + Namespace: novaAPI.Namespace, + Name: "internal-tls-certs"}, + PublicCertSecretName: types.NamespacedName{ + Namespace: novaAPI.Namespace, + Name: "public-tls-certs"}, + CaBundleSecretName: types.NamespacedName{ + Namespace: novaName.Namespace, + Name: "combined-ca-bundle"}, + InternalAPINetworkNADName: types.NamespacedName{ Namespace: novaName.Namespace, Name: "internalapi", diff --git a/test/functional/nova_compute_ironic_controller_test.go b/test/functional/nova_compute_ironic_controller_test.go index a4fe7f466..ebe6807d5 100644 --- a/test/functional/nova_compute_ironic_controller_test.go +++ b/test/functional/nova_compute_ironic_controller_test.go @@ -17,6 +17,7 @@ package functional_test import ( "encoding/json" + "fmt" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" . "github.com/onsi/ginkgo/v2" @@ -529,3 +530,64 @@ var _ = Describe("NovaCompute with ironic diver controller", func() { }) }) }) + +var _ = Describe("NovaCompute with ironic diver controller", func() { + When("NovaCompute is created with TLS CA cert secret", func() { + BeforeEach(func() { + spec := GetDefaultNovaComputeSpec(cell1) + spec["tls"] = map[string]interface{}{ + "caBundleSecretName": novaNames.CaBundleSecretName.Name, + } + novaCompute := CreateNovaCompute(cell1.NovaComputeName, spec) + DeferCleanup(th.DeleteInstance, novaCompute) + DeferCleanup( + k8sClient.Delete, + ctx, + CreateCellInternalSecret(cell1), + ) + }) + + It("reports that the CA secret is missing", func() { + th.ExpectConditionWithDetails( + cell1.NovaComputeName, + ConditionGetterFunc(NovaComputeConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/combined-ca-bundle not found", novaNames.Namespace), + ) + th.ExpectCondition( + cell1.NovaComputeName, + ConditionGetterFunc(NovaComputeConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("creates a StatefulSet for nova-compute service with TLS CA cert attached", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + th.SimulateStatefulSetReplicaReady(cell1.NovaComputeStatefulSetName) + + ss := th.GetStatefulSet(cell1.NovaComputeStatefulSetName) + + // Check the resulting deployment fields + Expect(int(*ss.Spec.Replicas)).To(Equal(1)) + Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(2)) + Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(1)) + + // cert deployment volumes + th.AssertVolumeExists(novaNames.CaBundleSecretName.Name, ss.Spec.Template.Spec.Volumes) + + // CA container certs + apiContainer := ss.Spec.Template.Spec.Containers[0] + th.AssertVolumeMountExists(novaNames.CaBundleSecretName.Name, "tls-ca-bundle.pem", apiContainer.VolumeMounts) + + th.ExpectCondition( + cell1.NovaComputeName, + ConditionGetterFunc(NovaComputeConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + }) +}) diff --git a/test/functional/nova_metadata_controller_test.go b/test/functional/nova_metadata_controller_test.go index e4bfad1ee..46cec8ca8 100644 --- a/test/functional/nova_metadata_controller_test.go +++ b/test/functional/nova_metadata_controller_test.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/util" @@ -734,3 +735,102 @@ var _ = Describe("NovaMetadata controller", func() { }) }) }) + +var _ = Describe("NovaMetadata controller", func() { + When("NovaMetadata is created with TLS CA cert secret", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, CreateInternalTopLevelSecret(novaNames)) + + spec := GetDefaultNovaMetadataSpec(novaNames.InternalTopLevelSecretName) + spec["tls"] = map[string]interface{}{ + "secretName": ptr.To(novaNames.InternalCertSecretName.Name), + "caBundleSecretName": novaNames.CaBundleSecretName.Name, + } + + DeferCleanup(th.DeleteInstance, CreateNovaMetadata(novaNames.MetadataName, spec)) + }) + + It("reports that the CA secret is missing", func() { + th.ExpectConditionWithDetails( + novaNames.MetadataName, + ConditionGetterFunc(NovaMetadataConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/combined-ca-bundle not found", novaNames.Namespace), + ) + th.ExpectCondition( + novaNames.MetadataName, + ConditionGetterFunc(NovaMetadataConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("reports that the service cert secret is missing", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + + th.ExpectConditionWithDetails( + novaNames.MetadataName, + ConditionGetterFunc(NovaMetadataConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/internal-tls-certs not found", novaNames.Namespace), + ) + th.ExpectCondition( + novaNames.MetadataName, + ConditionGetterFunc(NovaMetadataConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("creates a StatefulSet for nova-metadata service with TLS CA cert attached", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(novaNames.InternalCertSecretName)) + th.SimulateStatefulSetReplicaReady(novaNames.MetadataStatefulSetName) + + ss := th.GetStatefulSet(novaNames.MetadataStatefulSetName) + + // Check the resulting deployment fields + Expect(int(*ss.Spec.Replicas)).To(Equal(1)) + Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(4)) + Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(2)) + + // cert deployment volumes + th.AssertVolumeExists(novaNames.CaBundleSecretName.Name, ss.Spec.Template.Spec.Volumes) + th.AssertVolumeExists("nova-metadata-tls-certs", ss.Spec.Template.Spec.Volumes) + + // CA container certs + apiContainer := ss.Spec.Template.Spec.Containers[1] + th.AssertVolumeMountExists(novaNames.CaBundleSecretName.Name, "tls-ca-bundle.pem", apiContainer.VolumeMounts) + th.AssertVolumeMountExists("nova-metadata-tls-certs", "tls.key", apiContainer.VolumeMounts) + th.AssertVolumeMountExists("nova-metadata-tls-certs", "tls.crt", apiContainer.VolumeMounts) + + th.ExpectCondition( + novaNames.MetadataName, + ConditionGetterFunc(NovaMetadataConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + configDataMap := th.GetSecret(novaNames.MetadataConfigDataName) + Expect(configDataMap).ShouldNot(BeNil()) + Expect(configDataMap.Data).Should(HaveKey("httpd.conf")) + Expect(configDataMap.Data).Should(HaveKey("ssl.conf")) + configData := string(configDataMap.Data["httpd.conf"]) + Expect(configData).Should(ContainSubstring("SSLEngine on")) + Expect(configData).Should(ContainSubstring("SSLCertificateFile \"/etc/pki/tls/certs/nova-metadata.crt\"")) + Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile \"/etc/pki/tls/private/nova-metadata.key\"")) + + computeConfigData := th.GetSecret(novaNames.MetadataNeutronConfigDataName) + Expect(computeConfigData).ShouldNot(BeNil()) + Expect(computeConfigData.Data).To(HaveLen(1)) + Expect(computeConfigData.Data).Should(HaveKey("05-nova-metadata.conf")) + configData = string(computeConfigData.Data["05-nova-metadata.conf"]) + Expect(configData).Should(ContainSubstring("nova_metadata_protocol = https")) + }) + }) +}) diff --git a/test/functional/nova_novncproxy_test.go b/test/functional/nova_novncproxy_test.go index af504788e..3fce85906 100644 --- a/test/functional/nova_novncproxy_test.go +++ b/test/functional/nova_novncproxy_test.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ) var _ = Describe("NovaNoVNCProxy controller", func() { @@ -643,3 +644,97 @@ var _ = Describe("NovaNoVNCProxy controller", func() { }) }) }) + +var _ = Describe("NovaNoVNCProxy controller", func() { + When("NovaNoVNCProxy is created with TLS CA cert secret", func() { + BeforeEach(func() { + spec := GetDefaultNovaNoVNCProxySpec(cell1) + spec["tls"] = map[string]interface{}{ + "secretName": ptr.To(novaNames.InternalCertSecretName.Name), + "caBundleSecretName": novaNames.CaBundleSecretName.Name, + } + DeferCleanup(th.DeleteInstance, CreateNovaNoVNCProxy(cell1.NoVNCProxyName, spec)) + DeferCleanup( + k8sClient.Delete, ctx, CreateCellInternalSecret(cell1)) + }) + + It("reports that the CA secret is missing", func() { + th.ExpectConditionWithDetails( + cell1.NoVNCProxyName, + ConditionGetterFunc(NoVNCProxyConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/combined-ca-bundle not found", novaNames.Namespace), + ) + th.ExpectCondition( + cell1.NoVNCProxyName, + ConditionGetterFunc(NoVNCProxyConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("reports that the service cert secret is missing", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + + th.ExpectConditionWithDetails( + cell1.NoVNCProxyName, + ConditionGetterFunc(NoVNCProxyConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/internal-tls-certs not found", novaNames.Namespace), + ) + th.ExpectCondition( + cell1.NoVNCProxyName, + ConditionGetterFunc(NoVNCProxyConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("creates a StatefulSet for nova-metadata service with TLS CA cert attached", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(novaNames.InternalCertSecretName)) + th.SimulateStatefulSetReplicaReady(cell1.NoVNCProxyStatefulSetName) + + ss := th.GetStatefulSet(cell1.NoVNCProxyStatefulSetName) + + // Check the resulting deployment fields + Expect(int(*ss.Spec.Replicas)).To(Equal(1)) + Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(1)) + + // cert deployment volumes + th.AssertVolumeExists(novaNames.CaBundleSecretName.Name, ss.Spec.Template.Spec.Volumes) + th.AssertVolumeExists("nova-novncproxy-tls-certs", ss.Spec.Template.Spec.Volumes) + + // CA container certs + apiContainer := ss.Spec.Template.Spec.Containers[0] + th.AssertVolumeMountExists(novaNames.CaBundleSecretName.Name, "tls-ca-bundle.pem", apiContainer.VolumeMounts) + th.AssertVolumeMountExists("nova-novncproxy-tls-certs", "tls.key", apiContainer.VolumeMounts) + th.AssertVolumeMountExists("nova-novncproxy-tls-certs", "tls.crt", apiContainer.VolumeMounts) + + th.ExpectCondition( + cell1.NoVNCProxyName, + ConditionGetterFunc(NoVNCProxyConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + configDataMap := th.GetSecret( + types.NamespacedName{ + Namespace: cell1.NoVNCProxyName.Namespace, + Name: fmt.Sprintf("%s-config-data", cell1.NoVNCProxyName.Name), + }, + ) + Expect(configDataMap).ShouldNot(BeNil()) + Expect(configDataMap.Data).Should(HaveKey("01-nova.conf")) + configData := string(configDataMap.Data["01-nova.conf"]) + Expect(configData).Should(ContainSubstring("ssl_only=true")) + Expect(configData).Should(ContainSubstring("cert=/etc/pki/tls/certs/nova-novncproxy.crt")) + Expect(configData).Should(ContainSubstring("key=/etc/pki/tls/private/nova-novncproxy.key")) + }) + }) +}) diff --git a/test/functional/nova_scheduler_test.go b/test/functional/nova_scheduler_test.go index 4251db7b9..539bda76b 100644 --- a/test/functional/nova_scheduler_test.go +++ b/test/functional/nova_scheduler_test.go @@ -597,3 +597,60 @@ var _ = Describe("NovaScheduler controller cleaning", func() { }) }) }) + +var _ = Describe("NovaScheduler controller", func() { + When("NovaScheduler is created with TLS CA cert secret", func() { + BeforeEach(func() { + spec := GetDefaultNovaSchedulerSpec(novaNames) + spec["tls"] = map[string]interface{}{ + "caBundleSecretName": novaNames.CaBundleSecretName.Name, + } + + DeferCleanup( + k8sClient.Delete, ctx, CreateInternalTopLevelSecret(novaNames)) + DeferCleanup(th.DeleteInstance, CreateNovaScheduler(novaNames.SchedulerName, spec)) + }) + + It("reports that the CA secret is missing", func() { + th.ExpectConditionWithDetails( + novaNames.SchedulerName, + ConditionGetterFunc(NovaSchedulerConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/combined-ca-bundle not found", novaNames.Namespace), + ) + th.ExpectCondition( + novaNames.SchedulerName, + ConditionGetterFunc(NovaSchedulerConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("creates a StatefulSet for nova-scheduler service with TLS CA cert attached", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + th.SimulateStatefulSetReplicaReady(novaNames.SchedulerStatefulSetName) + + ss := th.GetStatefulSet(novaNames.SchedulerStatefulSetName) + // Check the resulting deployment fields + Expect(int(*ss.Spec.Replicas)).To(Equal(1)) + Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(2)) + Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(1)) + + // cert deployment volumes + th.AssertVolumeExists(novaNames.CaBundleSecretName.Name, ss.Spec.Template.Spec.Volumes) + + // CA container certs + apiContainer := ss.Spec.Template.Spec.Containers[0] + th.AssertVolumeMountExists(novaNames.CaBundleSecretName.Name, "tls-ca-bundle.pem", apiContainer.VolumeMounts) + + th.ExpectCondition( + novaNames.SchedulerName, + ConditionGetterFunc(NovaSchedulerConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + }) +}) diff --git a/test/functional/novaapi_controller_test.go b/test/functional/novaapi_controller_test.go index 0a747738b..9bb1c3152 100644 --- a/test/functional/novaapi_controller_test.go +++ b/test/functional/novaapi_controller_test.go @@ -804,3 +804,144 @@ var _ = Describe("NovaAPI controller", func() { }) }) }) + +var _ = Describe("NovaAPI controller", func() { + When("NovaAPI is created with TLS cert secrets", func() { + BeforeEach(func() { + spec := GetDefaultNovaAPISpec(novaNames) + spec["tls"] = map[string]interface{}{ + "api": map[string]interface{}{ + "internal": map[string]interface{}{ + "secretName": novaNames.InternalCertSecretName.Name, + }, + "public": map[string]interface{}{ + "secretName": novaNames.PublicCertSecretName.Name, + }, + }, + "caBundleSecretName": novaNames.CaBundleSecretName.Name, + } + + DeferCleanup( + k8sClient.Delete, ctx, CreateInternalTopLevelSecret(novaNames)) + DeferCleanup(th.DeleteInstance, CreateNovaAPI(novaNames.APIName, spec)) + }) + + It("reports that the CA secret is missing", func() { + th.ExpectConditionWithDetails( + novaNames.APIName, + ConditionGetterFunc(NovaAPIConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/combined-ca-bundle not found", novaNames.Namespace), + ) + th.ExpectCondition( + novaNames.APIName, + ConditionGetterFunc(NovaAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("reports that the internal cert secret is missing", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + + th.ExpectConditionWithDetails( + novaNames.APIName, + ConditionGetterFunc(NovaAPIConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/internal-tls-certs not found", novaNames.Namespace), + ) + th.ExpectCondition( + novaNames.APIName, + ConditionGetterFunc(NovaAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("reports that the public cert secret is missing", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(novaNames.InternalCertSecretName)) + + th.ExpectConditionWithDetails( + novaNames.APIName, + ConditionGetterFunc(NovaAPIConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/public-tls-certs not found", novaNames.Namespace), + ) + th.ExpectCondition( + novaNames.APIName, + ConditionGetterFunc(NovaAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("creates a StatefulSet for nova-api service with TLS certs attached", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(novaNames.InternalCertSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(novaNames.PublicCertSecretName)) + th.SimulateStatefulSetReplicaReady(novaNames.APIStatefulSetName) + keystone.SimulateKeystoneEndpointReady(novaNames.APIKeystoneEndpointName) + + ss := th.GetStatefulSet(novaNames.APIStatefulSetName) + // Check the resulting deployment fields + Expect(int(*ss.Spec.Replicas)).To(Equal(1)) + Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(5)) + Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(2)) + + // cert deployment volumes + th.AssertVolumeExists(novaNames.CaBundleSecretName.Name, ss.Spec.Template.Spec.Volumes) + th.AssertVolumeExists(novaNames.InternalCertSecretName.Name, ss.Spec.Template.Spec.Volumes) + th.AssertVolumeExists(novaNames.PublicCertSecretName.Name, ss.Spec.Template.Spec.Volumes) + + // httpd container certs + apiContainer := ss.Spec.Template.Spec.Containers[1] + th.AssertVolumeMountExists(novaNames.InternalCertSecretName.Name, "tls.key", apiContainer.VolumeMounts) + th.AssertVolumeMountExists(novaNames.InternalCertSecretName.Name, "tls.crt", apiContainer.VolumeMounts) + th.AssertVolumeMountExists(novaNames.PublicCertSecretName.Name, "tls.key", apiContainer.VolumeMounts) + th.AssertVolumeMountExists(novaNames.PublicCertSecretName.Name, "tls.crt", apiContainer.VolumeMounts) + th.AssertVolumeMountExists(novaNames.CaBundleSecretName.Name, "tls-ca-bundle.pem", apiContainer.VolumeMounts) + + Expect(apiContainer.ReadinessProbe.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTPS)) + Expect(apiContainer.LivenessProbe.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTPS)) + Expect(apiContainer.StartupProbe.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTPS)) + + configDataMap := th.GetSecret(novaNames.APIConfigDataName) + Expect(configDataMap).ShouldNot(BeNil()) + Expect(configDataMap.Data).Should(HaveKey("httpd.conf")) + Expect(configDataMap.Data).Should(HaveKey("ssl.conf")) + configData := string(configDataMap.Data["httpd.conf"]) + Expect(configData).Should(ContainSubstring("SSLEngine on")) + Expect(configData).Should(ContainSubstring("SSLCertificateFile \"/etc/pki/tls/certs/internal.crt\"")) + Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile \"/etc/pki/tls/private/internal.key\"")) + Expect(configData).Should(ContainSubstring("SSLCertificateFile \"/etc/pki/tls/certs/public.crt\"")) + Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile \"/etc/pki/tls/private/public.key\"")) + }) + + It("TLS Endpoints are created", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(novaNames.InternalCertSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(novaNames.PublicCertSecretName)) + th.SimulateStatefulSetReplicaReady(novaNames.APIStatefulSetName) + keystone.SimulateKeystoneEndpointReady(novaNames.APIKeystoneEndpointName) + + keystoneEndpoint := keystone.GetKeystoneEndpoint(types.NamespacedName{Namespace: novaNames.APIName.Namespace, Name: "nova"}) + endpoints := keystoneEndpoint.Spec.Endpoints + Expect(endpoints).To(HaveKeyWithValue("public", string("https://nova-public."+novaNames.APIName.Namespace+".svc:8774/v2.1"))) + Expect(endpoints).To(HaveKeyWithValue("internal", "https://nova-internal."+novaNames.APIName.Namespace+".svc:8774/v2.1")) + + th.ExpectCondition( + novaNames.APIName, + ConditionGetterFunc(NovaAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + }) +}) diff --git a/test/functional/novaconductor_controller_test.go b/test/functional/novaconductor_controller_test.go index aae60525d..368983e39 100644 --- a/test/functional/novaconductor_controller_test.go +++ b/test/functional/novaconductor_controller_test.go @@ -685,3 +685,62 @@ var _ = Describe("NovaConductor controller cleaning", func() { }) }) }) + +var _ = Describe("NovaConductor controller", func() { + When("NovaConductor is created with TLS CA cert secret", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, CreateCellInternalSecret(cell0)) + + spec := GetDefaultNovaConductorSpec(cell0) + spec["tls"] = map[string]interface{}{ + "caBundleSecretName": novaNames.CaBundleSecretName.Name, + } + DeferCleanup(th.DeleteInstance, CreateNovaConductor(cell0.ConductorName, spec)) + }) + + It("reports that the CA secret is missing", func() { + th.ExpectConditionWithDetails( + cell0.ConductorName, + ConditionGetterFunc(NovaConductorConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf("TLSInput error occured in TLS sources Secret %s/combined-ca-bundle not found", novaNames.Namespace), + ) + th.ExpectCondition( + cell0.ConductorName, + ConditionGetterFunc(NovaConductorConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("creates a StatefulSet for nova-conductor service with TLS CA cert attached", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(novaNames.CaBundleSecretName)) + th.SimulateJobSuccess(cell0.DBSyncJobName) + th.SimulateStatefulSetReplicaReady(cell0.ConductorStatefulSetName) + + ss := th.GetStatefulSet(cell0.ConductorStatefulSetName) + + // Check the resulting deployment fields + Expect(int(*ss.Spec.Replicas)).To(Equal(1)) + Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(2)) + Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(1)) + + // cert deployment volumes + th.AssertVolumeExists(novaNames.CaBundleSecretName.Name, ss.Spec.Template.Spec.Volumes) + + // CA container certs + apiContainer := ss.Spec.Template.Spec.Containers[0] + th.AssertVolumeMountExists(novaNames.CaBundleSecretName.Name, "tls-ca-bundle.pem", apiContainer.VolumeMounts) + + th.ExpectCondition( + cell0.ConductorName, + ConditionGetterFunc(NovaConductorConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + }) +})