From 394ebd5f2faa5c8c6b40ad279ec224f6dfe4b3a9 Mon Sep 17 00:00:00 2001 From: Sheng Lin Date: Wed, 22 Jan 2025 17:39:03 -0500 Subject: [PATCH] Add objectStore and ExternalDatabase fields to datastore .spec Signed-off-by: Sheng Lin --- api/apps/v1alpha1/nemo_datastore_types.go | 188 ++++++++++------- api/apps/v1alpha1/zz_generated.deepcopy.go | 90 ++++++-- .../apps.nvidia.com_nemodatastores.yaml | 157 ++++++++------ .../bases/apps.nvidia.com_nemodatastores.yaml | 192 +++++++++++++----- .../samples/apps_v1alpha1_nimdatastore.yaml | 36 +++- .../crds/apps.nvidia.com_nemodatastores.yaml | 157 ++++++++------ .../controller/nemo_datastore_controller.go | 12 +- 7 files changed, 554 insertions(+), 278 deletions(-) diff --git a/api/apps/v1alpha1/nemo_datastore_types.go b/api/apps/v1alpha1/nemo_datastore_types.go index f80be4c0..c5b011ff 100644 --- a/api/apps/v1alpha1/nemo_datastore_types.go +++ b/api/apps/v1alpha1/nemo_datastore_types.go @@ -20,6 +20,7 @@ import ( "fmt" "maps" "os" + "strconv" rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types" utils "github.com/NVIDIA/k8s-nim-operator/internal/utils" @@ -79,7 +80,22 @@ type NemoDatastoreSpec struct { GroupID *int64 `json:"groupID,omitempty"` RuntimeClass string `json:"runtimeClass,omitempty"` - DataStoreParams NemoDatastoreParams `json:"dataStoreParams"` + // ObjectStore specifies the location and credentials for accessing the external Object Storage + ObjectStoreConfig ObjectStoreConfig `json:"objectStoreConfig"` // e.g. minio + // ExternalDatabase contains external PostgreSQL configuration + DatabaseConfig DatabaseConfig `json:"databaseConfig"` // e.g. postgres + // secrets contains the pre-requisite secrets that must be created before deploying the datastore CR + Secrets Secrets `json:"secrets"` + // PVC defines the PersistentVolumeClaim for the datastore + PVC *PersistentVolumeClaim `json:"pvc,omitempty"` +} + +type Secrets struct { + GiteaAdminSecret string `json:"giteaAdminSecret"` + LfsJwtSecret string `json:"lfsJwtSecret"` + DataStoreInitSecret string `json:"datastoreInitSecret"` + DataStoreConfigSecret string `json:"datastoreConfigSecret"` // config_environment.sh + DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"` } // NemoDatastoreStatus defines the observed state of NemoDatastore @@ -89,21 +105,32 @@ type NemoDatastoreStatus struct { State string `json:"state,omitempty"` } -type NemoDatastoreParams struct { - DBSecret string `json:"dbSecret"` - GiteaAdminSecret string `json:"giteaAdminSecret"` +type ObjectStoreConfig struct { // e.g. Minio, s3 + // ObjectStoreCredentials stores the configuration to retrieve the object store credentials + Credentials ObjectStoreCredentials `json:"credentials"` - ObjectStoreSecret string `json:"objStoreSecret"` - DataStoreSettingsSecret string `json:"datastoreSettingsSecret"` - LfsJwtSecret string `json:"lfsJwtSecret"` + // +kubebuilder:default:=true + ServeDirect bool `json:"serveDirect,omitempty"` - DataStoreInitSecret string `json:"datastoreInitSecret"` - DataStoreConfigSecret string `json:"datastoreConfigSecret"` - DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"` + // endpoint is the fully qualidfied object store endpoint + Endpoint string `json:"endpoint"` + // BucketName is the bucket where LFS files will be stored + BucketName string `json:"bucketName"` + // Region is the region where bucket is hosted + Region string `json:"region"` + // SSL enable ssl for object store transport + SSL bool `json:"ssl"` +} - SshEnabled bool `json:"sshEnabled"` +type ObjectStoreCredentials struct { + // User is the non-root username for a NEMO Service in the object store. + User string `json:"user"` - PVC *PersistentVolumeClaim `json:"pvc,omitempty"` + // SecretName is the name of the secret which has the object credentials for a NEMO service user. + SecretName string `json:"secretName"` + + // PasswordKey is the name of the key in the `CredentialsSecret` secret for the object store credentials. + PasswordKey string `json:"passwordKey"` } // +genclient @@ -134,9 +161,8 @@ type NemoDatastoreList struct { // Prefers pvc.Name if explicitly set by the user in the NemoDatastore instance func (n *NemoDatastore) GetPVCName() string { pvcName := fmt.Sprintf("%s-pvc", n.GetName()) - dsParam := n.Spec.DataStoreParams - if dsParam.PVC != nil && dsParam.PVC.Name != "" { - pvcName = dsParam.PVC.Name + if n.Spec.PVC != nil && n.Spec.PVC.Name != "" { + pvcName = n.Spec.PVC.Name } return pvcName } @@ -197,23 +223,16 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { Value: "/data/gitea/git", }, { - Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreKey", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, - }, - }, - }, + Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", + Value: n.Spec.ObjectStoreConfig.Credentials.User, }, { Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreSecret", + Key: n.Spec.ObjectStoreConfig.Credentials.PasswordKey, LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, + Name: n.Spec.ObjectStoreConfig.Credentials.SecretName, }, }, }, @@ -224,7 +243,7 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { SecretKeyRef: &corev1.SecretKeySelector{ Key: "jwtSecret", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.LfsJwtSecret, + Name: n.Spec.Secrets.LfsJwtSecret, }, }, }, @@ -233,9 +252,9 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { Name: "GITEA__DATABASE__PASSWD", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "postgresPassword", + Key: n.Spec.DatabaseConfig.Credentials.PasswordKey, LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DBSecret, + Name: n.Spec.DatabaseConfig.Credentials.SecretName, }, }, }, @@ -245,6 +264,9 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { } func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { + objStoreSetting := n.Spec.ObjectStoreConfig + dbSetting := n.Spec.DatabaseConfig + envVars := []corev1.EnvVar{ { Name: "GITEA_APP_INI", @@ -271,23 +293,16 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { Value: "/data/gitea/git", }, { - Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreKey", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, - }, - }, - }, + Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", + Value: objStoreSetting.Credentials.User, }, { Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreSecret", + Key: objStoreSetting.Credentials.PasswordKey, LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, + Name: objStoreSetting.Credentials.SecretName, }, }, }, @@ -298,7 +313,7 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { SecretKeyRef: &corev1.SecretKeySelector{ Key: "jwtSecret", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.LfsJwtSecret, + Name: n.Spec.Secrets.LfsJwtSecret, }, }, }, @@ -307,9 +322,9 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { Name: "GITEA__DATABASE__PASSWD", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "postgresPassword", + Key: dbSetting.Credentials.PasswordKey, LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DBSecret, + Name: dbSetting.Credentials.SecretName, }, }, }, @@ -320,7 +335,7 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { SecretKeyRef: &corev1.SecretKeySelector{ Key: "GITEA_ADMIN_USERNAME", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.GiteaAdminSecret, + Name: n.Spec.Secrets.GiteaAdminSecret, }, }, }, @@ -331,11 +346,55 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { SecretKeyRef: &corev1.SecretKeySelector{ Key: "GITEA_ADMIN_PASSWORD", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.GiteaAdminSecret, + Name: n.Spec.Secrets.GiteaAdminSecret, }, }, }, }, + { + Name: "GITEA__LFS__SERVE_DIRECT", + Value: strconv.FormatBool(objStoreSetting.ServeDirect), + }, + { + Name: "GITEA__LFS__STORAGE_TYPE", + Value: "minio", + }, + { + Name: "GITEA__LFS__MINIO_ENDPOINT", + Value: objStoreSetting.Endpoint, + }, + { + Name: "GITEA__LFS__MINIO_BUCKET", + Value: objStoreSetting.BucketName, + }, + { + Name: "GITEA__LFS__MINIO_LOCATION", + Value: objStoreSetting.Region, + }, + { + Name: "GITEA__LFS__MINIO_LOCATION", + Value: objStoreSetting.Region, + }, + { + Name: "GITEA__LFS__MINIO_USE_SSL", + Value: strconv.FormatBool(objStoreSetting.SSL), + }, + { + Name: "GITEA__DATABASE__SSL_MODE", + Value: "disable", + }, + { + Name: "GITEA__DATABASE__NAME", + Value: dbSetting.DatabaseName, + }, + { + Name: "GITEA__DATABASE__HOST", + Value: fmt.Sprintf("%s:%d", dbSetting.Host, dbSetting.Port), + }, + { + Name: "GITEA__DATABASE__USER", + Value: dbSetting.Credentials.User, + }, } return envVars } @@ -369,7 +428,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { Name: "init", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreInitSecret, + SecretName: n.Spec.Secrets.DataStoreInitSecret, DefaultMode: &initMode, }, }, @@ -378,7 +437,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { Name: "config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreConfigSecret, + SecretName: n.Spec.Secrets.DataStoreConfigSecret, DefaultMode: &initMode, }, }, @@ -387,7 +446,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { Name: "inline-config-sources", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreInlineConfigSecret, + SecretName: n.Spec.Secrets.DataStoreInlineConfigSecret, DefaultMode: &configMode, }, }, @@ -400,7 +459,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { }, } - if n.Spec.DataStoreParams.PVC != nil { + if n.Spec.PVC != nil { volumes = append(volumes, corev1.Volume{ Name: "data", VolumeSource: corev1.VolumeSource{ @@ -421,25 +480,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { } func (n *NemoDatastore) ShouldCreatePersistentStorage() bool { - return n.Spec.DataStoreParams.PVC != nil && n.Spec.DataStoreParams.PVC.Create != nil && *n.Spec.DataStoreParams.PVC.Create -} - -// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance -func (n *NemoDatastore) GetEnvFrom() []corev1.EnvFromSource { - return []corev1.EnvFromSource{} -} - -// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance -func (n *NemoDatastore) GetInitAppIniEnvFrom() []corev1.EnvFromSource { - return []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DataStoreSettingsSecret, - }, - }, - }, - } + return n.Spec.PVC != nil && n.Spec.PVC.Create != nil && *n.Spec.PVC.Create } // GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance @@ -633,8 +674,8 @@ func (n *NemoDatastore) GetVolumeMounts() []corev1.VolumeMount { Name: "data", } - if n.Spec.DataStoreParams.PVC != nil { - dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath + if n.Spec.PVC != nil { + dataMount.SubPath = n.Spec.PVC.SubPath } mounts = append(mounts, dataMount) return mounts @@ -664,8 +705,8 @@ func (n *NemoDatastore) GetVolumeMountsInitContainer() []corev1.VolumeMount { Name: "data", } - if n.Spec.DataStoreParams.PVC != nil { - dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath + if n.Spec.PVC != nil { + dataMount.SubPath = n.Spec.PVC.SubPath } mounts = append(mounts, dataMount) return mounts @@ -682,7 +723,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container { }, VolumeMounts: n.GetVolumeMountsInitContainer(), Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), }, { Name: "init-app-ini", @@ -693,7 +733,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container { }, VolumeMounts: n.GetVolumeMountsInitContainer(), Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), }, { Name: "configure-datastore", @@ -707,7 +746,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container { }, VolumeMounts: n.GetVolumeMountsInitContainer(), Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), SecurityContext: &corev1.SecurityContext{ RunAsUser: n.GetUserID(), }, diff --git a/api/apps/v1alpha1/zz_generated.deepcopy.go b/api/apps/v1alpha1/zz_generated.deepcopy.go index d67b4c0b..69e1943f 100644 --- a/api/apps/v1alpha1/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/zz_generated.deepcopy.go @@ -182,6 +182,21 @@ func (in *Expose) DeepCopy() *Expose { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalDatabase) DeepCopyInto(out *ExternalDatabase) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalDatabase. +func (in *ExternalDatabase) DeepCopy() *ExternalDatabase { + if in == nil { + return nil + } + out := new(ExternalDatabase) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GPUSpec) DeepCopyInto(out *GPUSpec) { *out = *in @@ -1042,26 +1057,6 @@ func (in *NemoDatastoreList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreParams) DeepCopyInto(out *NemoDatastoreParams) { - *out = *in - if in.PVC != nil { - in, out := &in.PVC, &out.PVC - *out = new(PersistentVolumeClaim) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreParams. -func (in *NemoDatastoreParams) DeepCopy() *NemoDatastoreParams { - if in == nil { - return nil - } - out := new(NemoDatastoreParams) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NemoDatastoreSpec) DeepCopyInto(out *NemoDatastoreSpec) { *out = *in @@ -1137,7 +1132,14 @@ func (in *NemoDatastoreSpec) DeepCopyInto(out *NemoDatastoreSpec) { *out = new(int64) **out = **in } - in.DataStoreParams.DeepCopyInto(&out.DataStoreParams) + out.ObjectStoreConfig = in.ObjectStoreConfig + in.DatabaseConfig.DeepCopyInto(&out.DatabaseConfig) + out.Secrets = in.Secrets + if in.PVC != nil { + in, out := &in.PVC, &out.PVC + *out = new(PersistentVolumeClaim) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreSpec. @@ -1702,6 +1704,37 @@ func (in *NemoGuardrailStatus) DeepCopy() *NemoGuardrailStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreConfig) DeepCopyInto(out *ObjectStoreConfig) { + *out = *in + out.Credentials = in.Credentials +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreConfig. +func (in *ObjectStoreConfig) DeepCopy() *ObjectStoreConfig { + if in == nil { + return nil + } + out := new(ObjectStoreConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreCredentials) DeepCopyInto(out *ObjectStoreCredentials) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreCredentials. +func (in *ObjectStoreCredentials) DeepCopy() *ObjectStoreCredentials { + if in == nil { + return nil + } + out := new(ObjectStoreCredentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PersistentVolumeClaim) DeepCopyInto(out *PersistentVolumeClaim) { *out = *in @@ -1764,6 +1797,21 @@ func (in *Resources) DeepCopy() *Resources { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Secrets) DeepCopyInto(out *Secrets) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Secrets. +func (in *Secrets) DeepCopy() *Secrets { + if in == nil { + return nil + } + out := new(Secrets) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Service) DeepCopyInto(out *Service) { *out = *in diff --git a/bundle/manifests/apps.nvidia.com_nemodatastores.yaml b/bundle/manifests/apps.nvidia.com_nemodatastores.yaml index ae998d7d..4668139e 100644 --- a/bundle/manifests/apps.nvidia.com_nemodatastores.yaml +++ b/bundle/manifests/apps.nvidia.com_nemodatastores.yaml @@ -63,63 +63,6 @@ spec: items: type: string type: array - dataStoreParams: - properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: - properties: - accessModes: - items: - type: string - type: array - claimName: - type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: - type: string - storageClass: - type: string - required: - - claimName - - create - - enabled - - size - type: object - sshEnabled: - type: boolean - required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled - type: object env: items: description: EnvVar represents an environment variable present in @@ -545,6 +488,31 @@ spec: - port type: object type: object + externalDatabase: + properties: + database: + type: string + databaseSecret: + type: string + databaseSecretKey: + type: string + host: + type: string + port: + type: integer + sslMode: + type: string + user: + type: string + required: + - database + - databaseSecret + - databaseSecretKey + - host + - port + - sslMode + - user + type: object groupID: format: int64 type: integer @@ -764,6 +732,34 @@ spec: additionalProperties: type: string type: object + objectStore: + properties: + bucketName: + type: string + endpoint: + type: string + objectStoreSecret: + type: string + objectStoreSecretAccessKey: + type: string + objectStoreSecretAccessSecret: + type: string + region: + type: string + serveDirect: + type: boolean + ssl: + type: boolean + required: + - bucketName + - endpoint + - objectStoreSecret + - objectStoreSecretAccessKey + - objectStoreSecretAccessSecret + - region + - serveDirect + - ssl + type: object podAffinity: description: Pod affinity is a group of inter pod affinity scheduling rules. @@ -1120,6 +1116,30 @@ spec: type: array x-kubernetes-list-type: atomic type: object + pvc: + description: PersistentVolumeClaim defines the attributes of PVC used + as a source for caching NIM model + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during PVC creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. Leave it + as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access mode of the + PVC + type: string + type: object readinessProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -1952,6 +1972,25 @@ spec: - maxReplicas type: object type: object + secrets: + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - giteaAdminSecret + - lfsJwtSecret + type: object startupProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -2156,7 +2195,9 @@ spec: type: integer required: - authSecret - - dataStoreParams + - externalDatabase + - objectStore + - secrets type: object status: description: NemoDatastoreStatus defines the observed state of NemoDatastore diff --git a/config/crd/bases/apps.nvidia.com_nemodatastores.yaml b/config/crd/bases/apps.nvidia.com_nemodatastores.yaml index ae998d7d..d28fc3a0 100644 --- a/config/crd/bases/apps.nvidia.com_nemodatastores.yaml +++ b/config/crd/bases/apps.nvidia.com_nemodatastores.yaml @@ -63,62 +63,62 @@ spec: items: type: string type: array - dataStoreParams: + databaseConfig: + description: ExternalDatabase contains external PostgreSQL configuration properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: + credentials: + description: |- + DatabaseCredentials stores the configuration to retrieve the database credentials. + Required, must not be nil. properties: - accessModes: - items: - type: string - type: array - claimName: + passwordKey: + default: password + description: |- + PasswordKey is the name of the key in the `CredentialsSecret` secret for the database credentials. + Defaults to "password". type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: + secretName: + description: |- + SecretName is the name of the secret which has the database credentials for a NEMO service user. + Required, must not be empty. + minLength: 1 type: string - storageClass: + user: + description: |- + User is the non-root username for a NEMO Service in the database. + Required, must not be empty. + minLength: 1 type: string required: - - claimName - - create - - enabled - - size + - secretName + - user type: object - sshEnabled: - type: boolean + databaseName: + description: |- + DatabaseName is the database name for a NEMO Service. + Required, must not be empty. + minLength: 1 + type: string + host: + description: |- + Host is the hostname of the database. + Required, must not be empty. + minLength: 1 + type: string + port: + default: 5432 + description: |- + Port is the port where the database is reachable at. + If specified, this must be a valid port number, 0 < databasePort < 65536. + Defaults to 5432. + format: int32 + maximum: 65535 + minimum: 1 + type: integer required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled + - credentials + - databaseName + - host type: object env: items: @@ -764,6 +764,54 @@ spec: additionalProperties: type: string type: object + objectStoreConfig: + description: ObjectStore specifies the location and credentials for + accessing the external Object Storage + properties: + bucketName: + description: BucketName is the bucket where LFS files will be + stored + type: string + credentials: + description: ObjectStoreCredentials stores the configuration to + retrieve the object store credentials + properties: + passwordKey: + description: PasswordKey is the name of the key in the `CredentialsSecret` + secret for the object store credentials. + type: string + secretName: + description: SecretName is the name of the secret which has + the object credentials for a NEMO service user. + type: string + user: + description: User is the non-root username for a NEMO Service + in the object store. + type: string + required: + - passwordKey + - secretName + - user + type: object + endpoint: + description: endpoint is the fully qualidfied object store endpoint + type: string + region: + description: Region is the region where bucket is hosted + type: string + serveDirect: + default: true + type: boolean + ssl: + description: SSL enable ssl for object store transport + type: boolean + required: + - bucketName + - credentials + - endpoint + - region + - ssl + type: object podAffinity: description: Pod affinity is a group of inter pod affinity scheduling rules. @@ -1120,6 +1168,29 @@ spec: type: array x-kubernetes-list-type: atomic type: object + pvc: + description: PVC defines the PersistentVolumeClaim for the datastore + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during PVC creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. Leave it + as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access mode of the + PVC + type: string + type: object readinessProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -1952,6 +2023,27 @@ spec: - maxReplicas type: object type: object + secrets: + description: secrets contains the pre-requisite secrets that must + be created before deploying the datastore CR + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - giteaAdminSecret + - lfsJwtSecret + type: object startupProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -2156,7 +2248,9 @@ spec: type: integer required: - authSecret - - dataStoreParams + - databaseConfig + - objectStoreConfig + - secrets type: object status: description: NemoDatastoreStatus defines the observed state of NemoDatastore diff --git a/config/samples/apps_v1alpha1_nimdatastore.yaml b/config/samples/apps_v1alpha1_nimdatastore.yaml index c39ef799..01e2f34c 100644 --- a/config/samples/apps_v1alpha1_nimdatastore.yaml +++ b/config/samples/apps_v1alpha1_nimdatastore.yaml @@ -4,22 +4,36 @@ metadata: name: nemodatastore spec: authSecret: ngc-image-pull-secret - dataStoreParams: + secrets: datastoreConfigSecret: "nemo-ms-nemo-datastore" datastoreInitSecret: "nemo-ms-nemo-datastore-init" datastoreInlineConfigSecret: "nemo-ms-nemo-datastore-inline-config" - datastoreSettingsSecret: "nemo-ms-nemo-datastore-setting" - dbSecret: "nds-pg-existing-secret" giteaAdminSecret: "gitea-admin-credentials" lfsJwtSecret: "nemo-ms-nemo-datastore--lfs-jwt" - objStoreSecret: "nds-minio-existing-secret" - sshEnabled: false - pvc: - name: "pvc-shared-data" - create: true - storageClass: "local-path" - volumeAccessMode: ReadWriteOnce - size: "10Gi" + objectStoreConfig: + credentials: + user: minioUser + secretName: nds-minio-existing-secret + passwordKey: objectStoreSecret + serveDirect: true + endpoint: minio.k8s-nim-operator-system.svc.cluster.local:9000 + bucketName: datastore-dev + region: object-store-region + ssl: false + databaseConfig: + credentials: + user: ndsuser + secretName: nds-pg-existing-secret + passwordKey: postgresPassword + host: nds-pg-postgresql + port: 5432 + databaseName: ndsdb + pvc: + name: "pvc-shared-data" + create: true + storageClass: "local-path" + volumeAccessMode: ReadWriteOnce + size: "10Gi" expose: service: port: 3000 diff --git a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml index ae998d7d..4668139e 100644 --- a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml +++ b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml @@ -63,63 +63,6 @@ spec: items: type: string type: array - dataStoreParams: - properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: - properties: - accessModes: - items: - type: string - type: array - claimName: - type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: - type: string - storageClass: - type: string - required: - - claimName - - create - - enabled - - size - type: object - sshEnabled: - type: boolean - required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled - type: object env: items: description: EnvVar represents an environment variable present in @@ -545,6 +488,31 @@ spec: - port type: object type: object + externalDatabase: + properties: + database: + type: string + databaseSecret: + type: string + databaseSecretKey: + type: string + host: + type: string + port: + type: integer + sslMode: + type: string + user: + type: string + required: + - database + - databaseSecret + - databaseSecretKey + - host + - port + - sslMode + - user + type: object groupID: format: int64 type: integer @@ -764,6 +732,34 @@ spec: additionalProperties: type: string type: object + objectStore: + properties: + bucketName: + type: string + endpoint: + type: string + objectStoreSecret: + type: string + objectStoreSecretAccessKey: + type: string + objectStoreSecretAccessSecret: + type: string + region: + type: string + serveDirect: + type: boolean + ssl: + type: boolean + required: + - bucketName + - endpoint + - objectStoreSecret + - objectStoreSecretAccessKey + - objectStoreSecretAccessSecret + - region + - serveDirect + - ssl + type: object podAffinity: description: Pod affinity is a group of inter pod affinity scheduling rules. @@ -1120,6 +1116,30 @@ spec: type: array x-kubernetes-list-type: atomic type: object + pvc: + description: PersistentVolumeClaim defines the attributes of PVC used + as a source for caching NIM model + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during PVC creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. Leave it + as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access mode of the + PVC + type: string + type: object readinessProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -1952,6 +1972,25 @@ spec: - maxReplicas type: object type: object + secrets: + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - giteaAdminSecret + - lfsJwtSecret + type: object startupProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -2156,7 +2195,9 @@ spec: type: integer required: - authSecret - - dataStoreParams + - externalDatabase + - objectStore + - secrets type: object status: description: NemoDatastoreStatus defines the observed state of NemoDatastore diff --git a/internal/controller/nemo_datastore_controller.go b/internal/controller/nemo_datastore_controller.go index d9523dea..11f79c2b 100644 --- a/internal/controller/nemo_datastore_controller.go +++ b/internal/controller/nemo_datastore_controller.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -378,13 +379,12 @@ func (r *NemoDatastoreReconciler) reconcileNemoDatastore(ctx context.Context, ne if len(initContainers) > 0 { result.Spec.Template.Spec.InitContainers = initContainers } - envFrom := nemoDatastore.GetEnvFrom() - if len(envFrom) > 0 { - result.Spec.Template.Spec.Containers[0].EnvFrom = envFrom + fsGroup := ptr.To[int64](1000) + if nemoDatastore.Spec.GroupID != nil { + fsGroup = nemoDatastore.Spec.GroupID } - fsGroup := int64(1000) result.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ - FSGroup: &fsGroup, + FSGroup: fsGroup, } return result, nil }, "deployment", conditions.ReasonDeploymentFailed) @@ -431,7 +431,7 @@ func (r *NemoDatastoreReconciler) reconcilePVC(ctx context.Context, nemoDatastor // If PVC does not exist, create a new one if creation flag is enabled if err != nil { if nemoDatastore.ShouldCreatePersistentStorage() { - pvc, err = shared.ConstructPVC(*nemoDatastore.Spec.DataStoreParams.PVC, metav1.ObjectMeta{Name: pvcName, Namespace: nemoDatastore.GetNamespace()}) + pvc, err = shared.ConstructPVC(*nemoDatastore.Spec.PVC, metav1.ObjectMeta{Name: pvcName, Namespace: nemoDatastore.GetNamespace()}) if err != nil { logger.Error(err, "Failed to construct pvc", "name", pvcName) return err