Merge branch 'master' into propagate-annotations

Author: kobzonega

.github/workflows/run-tests.yml

@@ -163,7 +163,7 @@ jobs:
           kind load docker-image cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17
       - name: run-tests
         run: |
-          go test -v -timeout 1800s -p 1 ./... -args -ginkgo.v
+          go test -v -timeout 3600s -p 1 ./... -args -ginkgo.v
       - name: teardown-k8s-cluster
         run: |
           kind delete cluster

Makefile

@@ -79,7 +79,7 @@ kind-load:
 
 .PHONY: unit-test
 unit-test: manifests generate fmt vet envtest ## Run unit tests
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/controllers/... -ginkgo.v -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/... -ginkgo.v -coverprofile cover.out
 
 .PHONY: e2e-test
 e2e-test: manifests generate fmt vet docker-build kind-init kind-load ## Run e2e tests

api/v1alpha1/configuration.go

@@ -1,6 +1,7 @@
 package v1alpha1
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"fmt"
 	"path"
@@ -24,7 +25,7 @@ func hash(text string) string {
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
-func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
+func generateHosts(cr *Storage) []schema.Host {
 	var hosts []schema.Host
 
 	for i := 0; i < int(cr.Spec.Nodes); i++ {
@@ -57,6 +58,10 @@ func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
 		}
 	}
 
+	return hosts
+}
+
+func generateKeyConfig(cr *Storage, crDB *Database) *schema.KeyConfig {
 	var keyConfig *schema.KeyConfig
 	if crDB != nil && crDB.Spec.Encryption != nil && crDB.Spec.Encryption.Enabled {
 		keyConfig = &schema.KeyConfig{
@@ -71,25 +76,10 @@ func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
 		}
 	}
 
-	return schema.Configuration{
-		Hosts:     hosts,
-		KeyConfig: keyConfig,
-	}
+	return keyConfig
 }
 
-func tryFillMissingSections(
-	resultConfig map[string]interface{},
-	generatedConfig schema.Configuration,
-) {
-	if resultConfig["hosts"] == nil {
-		resultConfig["hosts"] = generatedConfig.Hosts
-	}
-	if generatedConfig.KeyConfig != nil {
-		resultConfig["key_config"] = generatedConfig.KeyConfig
-	}
-}
-
-func BuildConfiguration(cr *Storage, crDB *Database) (string, error) {
+func BuildConfiguration(cr *Storage, crDB *Database) ([]byte, error) {
 	config := make(map[string]interface{})
 
 	// If any kind of configuration exists on Database object, then
@@ -103,18 +93,47 @@ func BuildConfiguration(cr *Storage, crDB *Database) (string, error) {
 		rawYamlConfiguration = cr.Spec.Configuration
 	}
 
-	err := yaml.Unmarshal([]byte(rawYamlConfiguration), &config)
+	dynconfig, err := ParseDynconfig(rawYamlConfiguration)
+	if err == nil {
+		if dynconfig.Config["hosts"] == nil {
+			hosts := generateHosts(cr)
+			dynconfig.Config["hosts"] = hosts
+		}
+
+		return yaml.Marshal(dynconfig)
+	}
+
+	err = yaml.Unmarshal([]byte(rawYamlConfiguration), &config)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	generatedConfig := generateSomeDefaults(cr, crDB)
-	tryFillMissingSections(config, generatedConfig)
+	if config["hosts"] == nil {
+		hosts := generateHosts(cr)
+		config["hosts"] = hosts
+	}
 
-	data, err := yaml.Marshal(config)
-	if err != nil {
-		return "", err
+	// Will be removed by YDBOPS-9692
+	keyConfig := generateKeyConfig(cr, crDB)
+	if keyConfig != nil {
+		config["key_config"] = keyConfig
 	}
 
-	return string(data), nil
+	return yaml.Marshal(config)
+}
+
+func ParseConfig(rawYamlConfiguration string) (schema.Configuration, error) {
+	config := schema.Configuration{}
+	dec := yaml.NewDecoder(bytes.NewReader([]byte(rawYamlConfiguration)))
+	dec.KnownFields(false)
+	err := dec.Decode(&config)
+	return config, err
+}
+
+func ParseDynconfig(rawYamlConfiguration string) (schema.Dynconfig, error) {
+	dynconfig := schema.Dynconfig{}
+	dec := yaml.NewDecoder(bytes.NewReader([]byte(rawYamlConfiguration)))
+	dec.KnownFields(true)
+	err := dec.Decode(&dynconfig)
+	return dynconfig, err
 }

api/v1alpha1/connection_types.go

@@ -5,8 +5,9 @@ import (
 )
 
 type ConnectionOptions struct {
-	AccessToken       *AccessTokenAuth       `json:"accessToken,omitempty"`
-	StaticCredentials *StaticCredentialsAuth `json:"staticCredentials,omitempty"`
+	AccessToken        *AccessTokenAuth       `json:"accessToken,omitempty"`
+	StaticCredentials  *StaticCredentialsAuth `json:"staticCredentials,omitempty"`
+	Oauth2TokenExhange *Oauth2TokenExchange   `json:"oauth2TokenExchange,omitempty"`
 }
 
 type AccessTokenAuth struct {
@@ -18,6 +19,24 @@ type StaticCredentialsAuth struct {
 	Password *CredentialSource `json:"password,omitempty"`
 }
 
+type Oauth2TokenExchange struct {
+	Endpoint   string            `json:"endpoint"`
+	PrivateKey *CredentialSource `json:"privateKey"`
+	JWTHeader  *JWTHeader        `json:",inline"`
+	JWTClaims  *JWTClaims        `json:",inline"`
+}
+
+type JWTHeader struct {
+	KeyID   string `json:"keyID,omitempty"`
+	SignAlg string `json:"signAlg,omitempty"`
+}
+type JWTClaims struct {
+	Issuer   string `json:"issuer,omitempty"`
+	Subject  string `json:"subject,omitempty"`
+	Audience string `json:"audience,omitempty"`
+	ID       string `json:"id,omitempty"`
+}
+
 type CredentialSource struct {
 	SecretKeyRef *corev1.SecretKeySelector `json:"secretKeyRef"`
 }

api/v1alpha1/database_types.go

@@ -268,5 +268,6 @@ func init() {
 func (r *Database) AnyCertificatesAdded() bool {
 	return len(r.Spec.CABundle) > 0 ||
 		r.Spec.Service.GRPC.TLSConfiguration.Enabled ||
-		r.Spec.Service.Interconnect.TLSConfiguration.Enabled
+		r.Spec.Service.Interconnect.TLSConfiguration.Enabled ||
+		r.Spec.Service.Status.TLSConfiguration.Enabled
 }

api/v1alpha1/database_webhook.go

@@ -59,6 +59,10 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
 	database := obj.(*Database)
 	databaselog.Info("default", "name", database.Name)
 
+	if !database.Spec.OperatorSync {
+		return nil
+	}
+
 	if database.Spec.StorageClusterRef.Namespace == "" {
 		database.Spec.StorageClusterRef.Namespace = database.Namespace
 	}
@@ -107,6 +111,10 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
 		database.Spec.Service.Datastreams.TLSConfiguration = &TLSConfiguration{Enabled: false}
 	}
 
+	if database.Spec.Service.Status.TLSConfiguration == nil {
+		database.Spec.Service.Status.TLSConfiguration = &TLSConfiguration{Enabled: false}
+	}
+
 	if database.Spec.Domain == "" {
 		database.Spec.Domain = DefaultDatabaseDomain
 	}
@@ -147,7 +155,7 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
 		if err != nil {
 			return err
 		}
-		database.Spec.Configuration = configuration
+		database.Spec.Configuration = string(configuration)
 	}
 
 	if database.Spec.AdditionalAnnotations == nil {

api/v1alpha1/service_types.go

@@ -32,6 +32,8 @@ type InterconnectService struct {
 
 type StatusService struct {
 	Service `json:""`
+
+	TLSConfiguration *TLSConfiguration `json:"tls,omitempty"`
 }
 
 type DatastreamsService struct {

api/v1alpha1/storage_types.go

@@ -242,5 +242,6 @@ func init() {
 func (r *Storage) AnyCertificatesAdded() bool {
 	return len(r.Spec.CABundle) > 0 ||
 		r.Spec.Service.GRPC.TLSConfiguration.Enabled ||
-		r.Spec.Service.Interconnect.TLSConfiguration.Enabled
+		r.Spec.Service.Interconnect.TLSConfiguration.Enabled ||
+		r.Spec.Service.Status.TLSConfiguration.Enabled
 }