Allow customize http vhost config using HttpdCustomization.CustomConfigSecret

This change allows to customize the httpd vhost config using this parameter
to specify a secret that contains service config data. The content of each
provided snippet gets rendered as a go template and placed into
/etc/httpd/conf/httpd_custom_<endpoint>_<key> .
At the end of the vhost config in the default httpd template these custom
configs get included using `Include conf/httpd_custom_<endpoint>_*`.

For information on how sections in httpd configuration get merged, check section
"How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging

All possible parameters which can be use in a template can be looked up
in the <service>-config-data secret of the service like:
$ oc get secret -n openstack placement-config-data -o json | jq -r .data.TemplateParameters | base64 -d

or in the running pod of the service in the file:
$ cat /var/lib/config-data/default/TemplateParameters

The content is a versioned dump of the parameters of the service operator, like:

~~~
DatabaseConnection: mysql+pymysql://<user>:<pwd>@hostname-for-openstack.openstack.svc/placement?read_default_file=/etc/my.cnf
KeystoneInternalURL: http://keystone-internal.openstack.svc:5000
KeystonePublicURL: http://keystone-public-openstack.testin
LogFile: /var/log/placement/placement-api.log
PlacementPassword: <pwd>
ServiceUser: placement
TimeOut: 60
VHosts:
  internal:
    Override: false
    ServerName: placement-internal.openstack.svc
    TLS: false
  public:
    Override: false
    ServerName: placement-public.openstack.svc
    TLS: false
~~~

Depends-On: openstack-k8s-operators/lib-common#591
Depends-On: openstack-k8s-operators/lib-common#593

Jira: https://issues.redhat.com/browse/OSPRH-13100

Signed-off-by: Martin Schuppert <[email protected]>

Author: stuggi

api/bases/placement.openstack.org_placementapis.yaml

@@ -84,6 +84,20 @@ spec:
                 description: DefaultConfigOverwrite - interface to overwrite default
                   config files like policy.yaml.
                 type: object
+              httpdCustomization:
+                description: HttpdCustomization - customize the httpd service
+                properties:
+                  customConfigSecret:
+                    description: |-
+                      CustomConfigSecret - customize the httpd vhost config using this parameter to specify
+                      a secret that contains service config data. The content of each provided snippet gets
+                      rendered as a go template and placed into /etc/httpd/conf/httpd_custom_<key> .
+                      In the default httpd template at the end of the vhost those custom configs get
+                      included using `Include conf/httpd_custom_<endpoint>_*`.
+                      For information on how sections in httpd configuration get merged, check section
+                      "How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging
+                    type: string
+                type: object
               networkAttachments:
                 description: NetworkAttachments is a list of NetworkAttachment resource
                   names to expose the services to the given network

api/go.mod

@@ -3,7 +3,7 @@ module github.com/openstack-k8s-operators/placement-operator/api
 go 1.21
 
 require (
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7
 	k8s.io/api v0.29.10
 	k8s.io/apimachinery v0.29.10
 	sigs.k8s.io/controller-runtime v0.17.6

api/go.sum

@@ -73,8 +73,8 @@ github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo
 github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
 github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
 github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e h1:hf4kVQBkyG79WcHBxdQ25QrDBbGFdarebS1Tc0Xclq4=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7 h1:vXHpH93PjbAgg5ZN6n5WmxkybVQOs0nhXvVw62o7aZs=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

api/v1beta1/placementapi_types.go

@@ -124,6 +124,10 @@ type PlacementAPISpecCore struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// TLS - Parameters related to the TLS
 	TLS tls.API `json:"tls,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	// HttpdCustomization - customize the httpd service
+	HttpdCustomization HttpdCustomization `json:"httpdCustomization,omitempty"`
 }
 
 // APIOverrideSpec to override the generated manifest of several child resources.
@@ -141,6 +145,19 @@ type PasswordSelector struct {
 	Service string `json:"service"`
 }
 
+// HttpdCustomization - customize the httpd service
+type HttpdCustomization struct {
+	// +kubebuilder:validation:Optional
+	// CustomConfigSecret - customize the httpd vhost config using this parameter to specify
+	// a secret that contains service config data. The content of each provided snippet gets
+	// rendered as a go template and placed into /etc/httpd/conf/httpd_custom_<key> .
+	// In the default httpd template at the end of the vhost those custom configs get
+	// included using `Include conf/httpd_custom_<endpoint>_*`.
+	// For information on how sections in httpd configuration get merged, check section
+	// "How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging
+	CustomConfigSecret *string `json:"customConfigSecret,omitempty"`
+}
+
 // PlacementAPIStatus defines the observed state of PlacementAPI
 type PlacementAPIStatus struct {
 	// ReadyCount of placement API instances

api/v1beta1/zz_generated.deepcopy.go

@@ -84,6 +84,20 @@ spec:
                 description: DefaultConfigOverwrite - interface to overwrite default
                   config files like policy.yaml.
                 type: object
+              httpdCustomization:
+                description: HttpdCustomization - customize the httpd service
+                properties:
+                  customConfigSecret:
+                    description: |-
+                      CustomConfigSecret - customize the httpd vhost config using this parameter to specify
+                      a secret that contains service config data. The content of each provided snippet gets
+                      rendered as a go template and placed into /etc/httpd/conf/httpd_custom_<key> .
+                      In the default httpd template at the end of the vhost those custom configs get
+                      included using `Include conf/httpd_custom_<endpoint>_*`.
+                      For information on how sections in httpd configuration get merged, check section
+                      "How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging
+                    type: string
+                type: object
               networkAttachments:
                 description: NetworkAttachments is a list of NetworkAttachment resource
                   names to expose the services to the given network

config/crd/bases/placement.openstack.org_placementapis.yaml

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"time"
 
+	"gopkg.in/yaml.v2"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -815,17 +816,19 @@ func (r *PlacementAPIReconciler) initConditions(
 
 // fields to index to reconcile when change
 const (
-	passwordSecretField     = ".spec.secret"
-	caBundleSecretNameField = ".spec.tls.caBundleSecretName"
-	tlsAPIInternalField     = ".spec.tls.api.internal.secretName"
-	tlsAPIPublicField       = ".spec.tls.api.public.secretName"
+	passwordSecretField                 = ".spec.secret"
+	caBundleSecretNameField             = ".spec.tls.caBundleSecretName"
+	tlsAPIInternalField                 = ".spec.tls.api.internal.secretName"
+	tlsAPIPublicField                   = ".spec.tls.api.public.secretName"
+	httpdCustomServiceConfigSecretField = ".spec.httpdCustomization.customServiceConfigSecret"
 )
 
 var allWatchFields = []string{
 	passwordSecretField,
 	caBundleSecretNameField,
 	tlsAPIInternalField,
 	tlsAPIPublicField,
+	httpdCustomServiceConfigSecretField,
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -878,6 +881,18 @@ func (r *PlacementAPIReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return err
 	}
 
+	// index httpdOverrideSecretField
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &placementv1.PlacementAPI{}, httpdCustomServiceConfigSecretField, func(rawObj client.Object) []string {
+		// Extract the secret name from the spec, if one is provided
+		cr := rawObj.(*placementv1.PlacementAPI)
+		if cr.Spec.HttpdCustomization.CustomConfigSecret == nil {
+			return nil
+		}
+		return []string{*cr.Spec.HttpdCustomization.CustomConfigSecret}
+	}); err != nil {
+		return err
+	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&placementv1.PlacementAPI{}).
 		Owns(&mariadbv1.MariaDBDatabase{}).
@@ -1242,7 +1257,7 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 		"KeystoneInternalURL": keystoneInternalURL,
 		"KeystonePublicURL":   keystonePublicURL,
 		"PlacementPassword":   string(ospSecret.Data[instance.Spec.PasswordSelectors.Service]),
-		"log_file":            "/var/log/placement/placement-api.log",
+		"LogFile":             "/var/log/placement/placement-api.log",
 		"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
 			databaseAccount.Spec.UserName,
 			string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]),
@@ -1251,7 +1266,16 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 		),
 	}
 
+	httpdOverrideSecret := &corev1.Secret{}
+	if instance.Spec.HttpdCustomization.CustomConfigSecret != nil && *instance.Spec.HttpdCustomization.CustomConfigSecret != "" {
+		httpdOverrideSecret, _, err = secret.GetSecret(ctx, h, *instance.Spec.HttpdCustomization.CustomConfigSecret, instance.Namespace)
+		if err != nil {
+			return err
+		}
+	}
+
 	// create httpd  vhost template parameters
+	customTemplates := map[string]string{}
 	httpdVhostConfig := map[string]interface{}{}
 	for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} {
 		endptConfig := map[string]interface{}{}
@@ -1262,11 +1286,28 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 			endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String())
 			endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String())
 		}
+
+		endptConfig["Override"] = false
+		if len(httpdOverrideSecret.Data) > 0 {
+			endptConfig["Override"] = true
+			for key, data := range httpdOverrideSecret.Data {
+				if len(data) > 0 {
+					customTemplates["httpd_custom_"+endpt.String()+"_"+key] = string(data)
+				}
+			}
+		}
 		httpdVhostConfig[endpt.String()] = endptConfig
 	}
 	templateParameters["VHosts"] = httpdVhostConfig
 	templateParameters["TimeOut"] = instance.Spec.APITimeout
 
+	// Marshal the templateParameters map to YAML
+	yamlData, err := yaml.Marshal(templateParameters)
+	if err != nil {
+		return fmt.Errorf("Error marshalling to YAML: %w", err)
+	}
+	customData[common.TemplateParameters] = string(yamlData)
+
 	extraTemplates := map[string]string{
 		"placement.conf": "placementapi/config/placement.conf",
 	}
@@ -1286,6 +1327,7 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 			Namespace:          instance.Namespace,
 			Type:               util.TemplateTypeConfig,
 			InstanceType:       instance.Kind,
+			StringTemplate:     customTemplates,
 			CustomData:         customData,
 			ConfigOptions:      templateParameters,
 			Labels:             cmLabels,

controllers/placementapi_controller.go

@@ -9,11 +9,12 @@ require (
 	github.com/onsi/ginkgo/v2 v2.20.1
 	github.com/onsi/gomega v1.34.1
 	github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241212135809-dc78e7221d12
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241216113837-d172b3ac0f4e
 	github.com/openstack-k8s-operators/mariadb-operator/api v0.5.1-0.20241212160155-4e7d8f749820
 	github.com/openstack-k8s-operators/placement-operator/api v0.3.1-0.20240216174613-3d349f26e681
 	go.uber.org/zap v1.27.0
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.29.10
 	k8s.io/apimachinery v0.29.10
 	k8s.io/client-go v0.29.10
@@ -70,7 +71,6 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.29.10 // indirect
 	k8s.io/component-base v0.29.10 // indirect

go.mod

@@ -80,8 +80,8 @@ github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6
 github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
 github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241212135809-dc78e7221d12 h1:37tN4oVifWqkerafFrx3DFDDTOOzn2H+c67WIQ1Vkss=
 github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241212135809-dc78e7221d12/go.mod h1:AZhHY6dZzGyG9iVOf1poD7pTS9c7ZG/f99Fg+GdFVEk=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e h1:hf4kVQBkyG79WcHBxdQ25QrDBbGFdarebS1Tc0Xclq4=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7 h1:vXHpH93PjbAgg5ZN6n5WmxkybVQOs0nhXvVw62o7aZs=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e h1:HFo4OqPY0x4ZQeaWI2YGonTXAGTQFt+rOEJlfZVhS7s=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:IASoGvp5QM/tBJUd/8i8uIjj4DBnI+64Ydh4r7pmnvA=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241216113837-d172b3ac0f4e h1:/iWDp3j+ET3gE5IjKHtdZaPd4SQyLHB/4L5jB16cV3I=

go.sum

@@ -68,6 +68,11 @@ CustomLog /dev/stdout proxy env=forwarded
   WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=placement processes=3 threads=1 user=placement
   WSGIProcessGroup {{ $endpt }}
   WSGIScriptAlias / /usr/bin/placement-api
+
+{{- if $vhost.Override }}
+  Include conf/httpd_custom_{{ $endpt }}_*
+{{- end }}
+
 </VirtualHost>
 {{ end }}
 

templates/placementapi/config/httpd.conf

@@ -53,6 +53,13 @@
             "dest": "/etc/my.cnf",
             "owner": "placement",
             "perm": "0644"
+        },
+        {
+            "source": "/var/lib/config-data/default/httpd_custom_*",
+            "dest": "/etc/httpd/conf/",
+            "owner": "apache",
+            "perm": "0444",
+            "optional": true
         }
     ],
     "permissions": [

templates/placementapi/config/placement-api-config.json

@@ -3,8 +3,8 @@
 max_logfile_count=5
 max_logfile_size_mb=50
 log_rotation_type=size
-{{if (index . "log_file") }}
-log_file = {{ .log_file }}
+{{if (index . "LogFile") }}
+log_file = {{ .LogFile }}
 {{end}}
 debug = true
 

templates/placementapi/config/placement.conf

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 
 	keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
+	"github.com/openstack-k8s-operators/lib-common/modules/common"
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	mariadb_test "github.com/openstack-k8s-operators/mariadb-operator/api/test/helpers"
 )
@@ -977,6 +978,62 @@ var _ = Describe("PlacementAPI controller", func() {
 			}, timeout, interval).Should(Succeed())
 		})
 	})
+
+	When("A PlacementAPI is created with HttpdCustomization.CustomConfigSecret", func() {
+		BeforeEach(func() {
+			customServiceConfigSecretName := types.NamespacedName{Name: "foo", Namespace: namespace}
+			customConfig := []byte(`CustomParam "foo"
+CustomKeystonePublicURL "{{ .KeystonePublicURL }}"`)
+			th.CreateSecret(
+				customServiceConfigSecretName,
+				map[string][]byte{
+					"bar.conf": customConfig,
+				},
+			)
+
+			spec := GetDefaultPlacementAPISpec()
+			spec["httpdCustomization"] = map[string]interface{}{
+				"customConfigSecret": customServiceConfigSecretName.Name,
+			}
+
+			placement := CreatePlacementAPI(names.PlacementAPIName, spec)
+			DeferCleanup(th.DeleteInstance, placement)
+
+			DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(namespace))
+			DeferCleanup(k8sClient.Delete, ctx, CreatePlacementAPISecret(namespace, SecretName))
+
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBAccount)
+
+			th.SimulateJobSuccess(names.DBSyncJobName)
+			th.SimulateDeploymentReplicaReady(names.DeploymentName)
+			keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName)
+			keystone.SimulateKeystoneEndpointReady(names.KeystoneEndpointName)
+			DeferCleanup(th.DeleteInstance, placement)
+		})
+
+		It("it renders the custom template and adds it to the placement-config-data secret", func() {
+			scrt := th.GetSecret(names.ConfigMapName)
+			Expect(scrt).ShouldNot(BeNil())
+			Expect(scrt.Data).Should(HaveKey(common.TemplateParameters))
+			configData := string(scrt.Data[common.TemplateParameters])
+			keystonePublicURL := "http://keystone-public-openstack.testing"
+			Expect(configData).Should(ContainSubstring(fmt.Sprintf("KeystonePublicURL: %s", keystonePublicURL)))
+
+			for _, cfg := range []string{"httpd_custom_internal_bar.conf", "httpd_custom_public_bar.conf"} {
+				Expect(scrt.Data).Should(HaveKey(cfg))
+				configData := string(scrt.Data[cfg])
+				Expect(configData).Should(ContainSubstring("CustomParam \"foo\""))
+				Expect(configData).Should(ContainSubstring(fmt.Sprintf("CustomKeystonePublicURL \"%s\"", keystonePublicURL)))
+			}
+		})
+	})
+
 	// Run MariaDBAccount suite tests.  these are pre-packaged ginkgo tests
 	// that exercise standard account create / update patterns that should be
 	// common to all controllers that ensure MariaDBAccount CRs.