Merge "zuul connections: store some sensitive parameters in secrets"

Author: Microzuul CI

api/v1/softwarefactory_types.go

@@ -207,10 +207,13 @@ type SMTPConnection struct {
 	DefaultTo string `json:"defaultTo,omitempty"`
 	// [user](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.user)
 	User string `json:"user,omitempty"`
-	// [password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password)
+	// DEPRECATED use `Secrets` instead to securely store this value [password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password)
 	Password string `json:"password,omitempty"`
 	// [use_starttls](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.use_starttls)
 	TLS *bool `json:"tls,omitempty"`
+	// Name of the secret which contains the following keys:
+	// the [password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password)
+	Secrets *string `json:"secrets,omitempty"`
 }
 
 // Describes a Zuul connection using the [ElasticSearch driver](https://zuul-ci.org/docs/zuul/latest/drivers/elasticsearch.html#connection-configuration).
@@ -224,6 +227,10 @@ type ElasticSearchConnection struct {
 	UseSSL *bool `json:"useSSL,omitempty"`
 	// [verifyCerts](https://zuul-ci.org/docs/zuul/latest/drivers/elasticsearch.html#attr-%3CElasticsearch%20connection%3E.verify_certs)
 	VerifyCerts *bool `json:"verifyCerts,omitempty"`
+	// If the connection requires basic authentication, the name of the secret containing the following keys:
+	// * username
+	// * password
+	BasicAuthSecret *string `json:"basicAuthSecret,omitempty"`
 }
 
 // The description of an OpenIDConnect authenticator, see [Zuul's authentication documentation](https://zuul-ci.org/docs/zuul/latest/configuration.html#authentication)

api/v1/zz_generated.deepcopy.go

@@ -511,6 +511,12 @@ spec:
                         Describes a Zuul connection using the [ElasticSearch driver](https://zuul-ci.org/docs/zuul/latest/drivers/elasticsearch.html#connection-configuration).
                         When an optional parameter is not specified then Zuul's defaults apply
                       properties:
+                        basicAuthSecret:
+                          description: |-
+                            If the connection requires basic authentication, the name of the secret containing the following keys:
+                            * username
+                            * password
+                          type: string
                         name:
                           description: How the connection will be named in Zuul's
                             configuration and appear in zuul-web
@@ -1104,11 +1110,17 @@ spec:
                             configuration and appear in zuul-web
                           type: string
                         password:
-                          description: '[password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password)'
+                          description: DEPRECATED use `Secrets` instead to securely
+                            store this value [password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password)
                           type: string
                         port:
                           description: '[port](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.port)'
                           type: integer
+                        secrets:
+                          description: |-
+                            Name of the secret which contains the following keys:
+                            the [password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password)
+                          type: string
                         server:
                           description: '[server](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.server)'
                           type: string

config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml

@@ -7,13 +7,15 @@ import (
 	"bytes"
 	_ "embed"
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 
 	"golang.org/x/exp/maps"
 	ini "gopkg.in/ini.v1"
 	appsv1 "k8s.io/api/apps/v1"
 	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
@@ -1331,10 +1333,37 @@ func AddGitConnection(cfg *ini.File, name string, baseurl string, poolDelay int3
 
 func (r *SFController) AddElasticSearchConnection(cfg *ini.File, conn sfv1.ElasticSearchConnection) {
 	section := "connection " + conn.Name
+	scheme := ""
+	uri := conn.URI
+	// crude clear-text basic auth check
+	if match, _ := regexp.MatchString("http[s]?://[^:]+:.+@.+", uri); match {
+		utils.LogI(fmt.Sprintf("It looks like elasticsearch connection %s has basic auth secrets stored in clear text. Use the 'basicAuthSecret' property instead", conn.Name))
+	}
+	if strings.HasPrefix(uri, "https://") {
+		scheme = "https://"
+		// TODO might not work with unicode URLs
+		uri = uri[len("https://"):]
+	}
+	if strings.HasPrefix(uri, "http://") {
+		scheme = "http://"
+		// TODO might not work with unicode URLs
+		uri = uri[len("http://"):]
+	}
+	if conn.BasicAuthSecret != nil {
+		password, passwordErr := r.GetSecretDataFromKey(*conn.BasicAuthSecret, "password")
+		// TODO we may also want to handle missing values in the secret
+		if errors.IsNotFound(passwordErr) {
+			utils.LogE(passwordErr, fmt.Sprintf("elasticsearch connection %s refers to a non-existing secret: %s ", conn.Name, *conn.BasicAuthSecret))
+		}
+		username, _ := r.GetSecretDataFromKey(*conn.BasicAuthSecret, "username")
+		uri = string(username) + ":" + string(password) + "@" + uri
+	}
+	uri = scheme + uri
+
 	cfg.NewSection(section)
 	cfg.Section(section).NewKey("driver", "elasticsearch")
 	cfg.Section(section).NewKey("ca_certs", "/etc/ssl/certs/ca-bundle.crt")
-	cfg.Section(section).NewKey("uri", conn.URI)
+	cfg.Section(section).NewKey("uri", uri)
 	// Optional fields (set as omitempty in ElasticSearchConnection struct definition)
 	if conn.UseSSL != nil && !*conn.UseSSL {
 		cfg.Section(section).NewKey("use_ssl", "false")
@@ -1362,8 +1391,17 @@ func (r *SFController) AddSMTPConnection(cfg *ini.File, conn sfv1.SMTPConnection
 	if conn.User != "" {
 		cfg.Section(section).NewKey("user", conn.User)
 	}
-	if conn.Password != "" {
-		cfg.Section(section).NewKey("password", conn.Password)
+	if conn.Secrets != nil {
+		password, passwordErr := r.GetSecretDataFromKey(*conn.Secrets, "password")
+		if errors.IsNotFound(passwordErr) {
+			utils.LogE(passwordErr, fmt.Sprintf("SMTP connection %s refers to a non-existing secret: %s ", conn.Name, *conn.Secrets))
+		}
+		cfg.Section(section).NewKey("password", string(password))
+	} else {
+		if conn.Password != "" {
+			utils.LogI("Deprecation Warning: SMTPConnection's Password field will disappear in a future version. Use Secrets instead")
+			cfg.Section(section).NewKey("password", conn.Password)
+		}
 	}
 	if conn.TLS != nil && !*conn.TLS {
 		cfg.Section(section).NewKey("use_starttls", "false")

controllers/zuul.go

@@ -6,13 +6,21 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
+- zuul: add `BasicAuthSecret` parameter for elasticsearch connections. This parameter
+  allows defining basic auth settings (username and password) and store them in a secret
+  rather than in plain text in the software factort manifest.
+
 ### Changed
 
 - The default CPU limits have been reduced from 2000m to 500m to enable rollout on smaller cluster.
 - go version in go.mod is bumped to 1.24. Backward compatibility with earlier version is not guaranted.
 - zuul-* : bumped to 11.3.0-20250414-1 (using ubi9 latest images)
 
 ### Deprecated
+
+- zuul: the `Password` parameter in SMTP connections is deprecated and will be removed
+  in a future version. Use instead `Secrets` to point to a secret holding the password.
+
 ### Removed
 ### Fixed
 

doc/reference/CHANGELOG.md

@@ -82,6 +82,7 @@ _Appears in:_
 | `uri` _string_ | [uri](https://zuul-ci.org/docs/zuul/latest/drivers/elasticsearch.html#attr-%3CElasticsearch%20connection%3E.uri) | -|
 | `useSSL` _boolean_ | [useSSL](https://zuul-ci.org/docs/zuul/latest/drivers/elasticsearch.html#attr-%3CElasticsearch%20connection%3E.use_ssl) | -|
 | `verifyCerts` _boolean_ | [verifyCerts](https://zuul-ci.org/docs/zuul/latest/drivers/elasticsearch.html#attr-%3CElasticsearch%20connection%3E.verify_certs) | -|
+| `basicAuthSecret` _string_ | If the connection requires basic authentication, the name of the secret containing the following keys: * username * password | -|
 
 
 #### FluentBitForwarderSpec
@@ -379,8 +380,9 @@ _Appears in:_
 | `defaultFrom` _string_ | [default_from](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.default_from) | -|
 | `defaultTo` _string_ | [default_to](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.default_to) | -|
 | `user` _string_ | [user](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.user) | -|
-| `password` _string_ | [password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password) | -|
+| `password` _string_ | DEPRECATED use `Secrets` instead to securely store this value [password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password) | -|
 | `tls` _boolean_ | [use_starttls](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.use_starttls) | -|
+| `secrets` _string_ | Name of the secret which contains the following keys: the [password](https://zuul-ci.org/docs/zuul/latest/drivers/smtp.html#attr-%3Csmtp%20connection%3E.password) | -|
 
 
 #### Secret

doc/reference/api/index.md

@@ -24,12 +24,39 @@
     dummy_elasticsearchconns:
       - name: dummy-elasticsearch-conn
         uri: http://test:9200
+        basicAuthSecret: es-basicauth
     dummy_pagureconns:
       - name: dummy-pagure-conn
         secrets: pagureconnectionsecret
     dummy_smtpconns:
       - name: dummy-smtp-conn
         server: smtp.domain.com
+        secrets: smtp-secret
+
+- name: Create SMTP Connection Secret
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: smtp-secret
+        namespace: sf
+      data:
+        password: "{{ 'smtp-password' | b64encode }}"
+
+- name: Create ElasticSearch Connection Secret
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: es-basicauth
+        namespace: sf
+      data:
+        username: "{{ 'es-username' | b64encode }}"
+        password: "{{ 'es-password' | b64encode }}"
 
 - name: Create Gerrit Connection Secret
   kubernetes.core.k8s:
@@ -135,6 +162,14 @@
         kubectl exec zuul-scheduler-0 -- grep "dummy-elasticsearch-conn" /etc/zuul/zuul.conf
         kubectl exec zuul-scheduler-0 -- grep "dummy-smtp-conn" /etc/zuul/zuul.conf
 
+    - name: Ensure ElasticSearch URI is configured with basic auth
+      ansible.builtin.shell: |
+        kubectl exec zuul-scheduler-0 -- grep "http://es-username:es-password@test:9200" /etc/zuul/zuul.conf
+
+    - name: Ensure SMTP password is present
+      ansible.builtin.shell: |
+        kubectl exec zuul-scheduler-0 -- grep "smtp-password" /etc/zuul/zuul.conf
+
     - name: Ensure the new Zuul dummy gerrit secret exist in the scheduler's zuul.conf
       ansible.builtin.shell: |
         set -e