From e380515b119f421fb7b6d6992cf54dc5e4a6b0fd Mon Sep 17 00:00:00 2001 From: shil Date: Mon, 23 Dec 2024 16:58:33 -0800 Subject: [PATCH 1/4] chore: temp work to support datastore v2 Signed-off-by: Sheng Lin --- api/apps/v1alpha1/nemo_datastorev2_types.go | 973 ++++++++ api/apps/v1alpha1/zz_generated.deepcopy.go | 184 ++ .../typed/apps/v1alpha1/apps_client.go | 5 + .../apps/v1alpha1/fake/fake_apps_client.go | 4 + .../v1alpha1/fake/fake_nemodatastorev2.go | 146 ++ .../apps/v1alpha1/generated_expansion.go | 2 + .../typed/apps/v1alpha1/nemodatastorev2.go | 68 + .../apps.nvidia.com_nemodatastorev2s.yaml | 2212 +++++++++++++++++ cmd/main.go | 11 + .../apps.nvidia.com_nemodatastorev2s.yaml | 2212 +++++++++++++++++ config/rbac/role.yaml | 3 + .../apps.nvidia.com_nemodatastorev2s.yaml | 2212 +++++++++++++++++ .../controller/nemo_datastorev2_controller.go | 642 +++++ manifests/deployment.yaml | 5 + 14 files changed, 8679 insertions(+) create mode 100644 api/apps/v1alpha1/nemo_datastorev2_types.go create mode 100644 api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go create mode 100644 api/versioned/typed/apps/v1alpha1/nemodatastorev2.go create mode 100644 bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml create mode 100644 config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml create mode 100644 deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml create mode 100644 internal/controller/nemo_datastorev2_controller.go diff --git a/api/apps/v1alpha1/nemo_datastorev2_types.go b/api/apps/v1alpha1/nemo_datastorev2_types.go new file mode 100644 index 00000000..c32aad68 --- /dev/null +++ b/api/apps/v1alpha1/nemo_datastorev2_types.go @@ -0,0 +1,973 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "fmt" + "maps" + "os" + + rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types" + utils "github.com/NVIDIA/k8s-nim-operator/internal/utils" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +const ( + // NemoDatastoreV2ConditionReady indicates that the NEMO datastore service is ready. + NemoDatastoreV2ConditionReady = "Ready" + // NemoDatastoreV2ConditionFailed indicates that the NEMO datastore service has failed. + NemoDatastoreV2ConditionFailed = "Failed" + + // NemoDatastoreV2StatusPending indicates that NEMO datastore service is in pending state + NemoDatastoreV2StatusPending = "Pending" + // NemoDatastoreV2StatusNotReady indicates that NEMO datastore service is not ready + NemoDatastoreV2StatusNotReady = "NotReady" + // NemoDatastoreV2StatusReady indicates that NEMO datastore service is ready + NemoDatastoreV2StatusReady = "Ready" + // NemoDatastoreV2StatusFailed indicates that NEMO datastore service has failed + NemoDatastoreV2StatusFailed = "Failed" +) + +// NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 +type NemoDatastoreV2Spec struct { + Image Image `json:"image,omitempty"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env []corev1.EnvVar `json:"env,omitempty"` + // The name of an secret that contains authn for the NGC NIM service API + AuthSecret string `json:"authSecret"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PodAffinity *corev1.PodAffinity `json:"podAffinity,omitempty"` + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + Expose Expose `json:"expose,omitempty"` + LivenessProbe Probe `json:"livenessProbe,omitempty"` + ReadinessProbe Probe `json:"readinessProbe,omitempty"` + StartupProbe Probe `json:"startupProbe,omitempty"` + Scale Autoscaling `json:"scale,omitempty"` + Metrics Metrics `json:"metrics,omitempty"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default:=1 + Replicas int `json:"replicas,omitempty"` + UserID *int64 `json:"userID,omitempty"` + GroupID *int64 `json:"groupID,omitempty"` + RuntimeClass string `json:"runtimeClass,omitempty"` + + DataStoreParams NemoDatastoreV2Params `json:"dataStoreParams"` +} + +// NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 +type NemoDatastoreV2Status struct { + Conditions []metav1.Condition `json:"conditions,omitempty"` + AvailableReplicas int32 `json:"availableReplicas,omitempty"` + State string `json:"state,omitempty"` +} + +type NemoDatastoreV2Params struct { + DBSecret string `json:"dbSecret"` + GiteaAdminSecret string `json:"giteaAdminSecret"` + + ObjectStoreSecret string `json:"objStoreSecret"` + DataStoreSettingsSecret string `json:"datastoreSettingsSecret"` + LfsJwtSecret string `json:"lfsJwtSecret"` + + DataStoreInitSecret string `json:"datastoreInitSecret"` + DataStoreConfigSecret string `json:"datastoreConfigSecret"` + DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"` + + SshEnabled bool `json:"sshEnabled"` + + PVCSharedData string `json:"pvcSharedData"` + StorageClass string `json:"storageClass"` + Size string `json:"size,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0 +// +kubebuilder:printcolumn:name="Age",type="date",format="date-time",JSONPath=".metadata.creationTimestamp",priority=0 + +// NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API +type NemoDatastoreV2 struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NemoDatastoreV2Spec `json:"spec,omitempty"` + Status NemoDatastoreV2Status `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NemoDatastoreV2List contains a list of NemoDatastoreV2 +type NemoDatastoreV2List struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NemoDatastoreV2 `json:"items"` +} + +// GetPVCName returns the name to be used for the PVC based on the custom spec +// Prefers pvc.Name if explicitly set by the user in the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetPVCName(pvc PersistentVolumeClaim) string { + pvcName := fmt.Sprintf("%s-pvc", n.GetName()) + if pvc.Name != "" { + pvcName = pvc.Name + } + return pvcName +} + +// GetStandardSelectorLabels returns the standard selector labels for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetStandardSelectorLabels() map[string]string { + return map[string]string{ + "app.kubernetes.io/name": n.Name, + "app.kubernetes.io/instance": n.Name, + } +} + +// GetStandardLabels returns the standard set of labels for NemoDatastoreV2 resources +func (n *NemoDatastoreV2) GetStandardLabels() map[string]string { + return map[string]string{ + "app.kubernetes.io/name": n.Name, + "app.kubernetes.io/instance": n.Name, + "app.kubernetes.io/operator-version": os.Getenv("OPERATOR_VERSION"), + "app.kubernetes.io/part-of": "nemo-datastore-service", + "app.kubernetes.io/managed-by": "k8s-nim-operator", + } +} + +// GetMainContainerEnv returns the standard set of env variables for the NemoDatastoreV2 main container +func (n *NemoDatastoreV2) GetStandardEnv() []corev1.EnvVar { + // add standard env required for NIM service + envVars := []corev1.EnvVar{ + { + Name: "SSH_LISTEN_PORT", + Value: "2222", + }, + { + Name: "SSH_PORT", + Value: "22", + }, + { + Name: "GITEA_APP_INI", + Value: "/data/gitea/conf/app.ini", + }, + { + Name: "GITEA_CUSTOM", + Value: "/data/gitea", + }, + { + Name: "GITEA_WORK_DIR", + Value: "/data", + }, + { + Name: "TMPDIR", + Value: "/tmp/gitea", + }, + { + Name: "GITEA_TEMP", + Value: "/tmp/gitea", + }, + { + Name: "HOME", + Value: "/data/gitea/git", + }, + { + Name: "GITEA__DATABASE__PASSWD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "password", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DBSecret, + }, + }, + }, + }, + } + return envVars +} + +func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar { + envVars := []corev1.EnvVar{ + { + Name: "GITEA_APP_INI", + Value: "/data/gitea/conf/app.ini", + }, + { + Name: "GITEA_CUSTOM", + Value: "/data/gitea", + }, + { + Name: "GITEA_WORK_DIR", + Value: "/data", + }, + { + Name: "TMPDIR", + Value: "/tmp/gitea", + }, + { + Name: "GITEA_TEMP", + Value: "/tmp/gitea", + }, + + { + Name: "HOME", + Value: "/data/gitea/git", + }, + { + Name: "GITEA__DATABASE__PASSWD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "password", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DBSecret, + }, + }, + }, + }, + { + Name: "GITEA_ADMIN_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "username", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.GiteaAdminSecret, + }, + }, + }, + }, + { + Name: "GITEA_ADMIN_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "password", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.GiteaAdminSecret, + }, + }, + }, + }, + } + return envVars +} + +// GetVolumes returns volumes for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetVolumes() []corev1.Volume { + /*volumes: + - name: init + secret: + defaultMode: 110 + secretName: datastore-v2-nemo-datastore-init + - name: config + secret: + defaultMode: 110 + secretName: datastore-v2-nemo-datastore + - name: inline-config-sources + secret: + defaultMode: 420 + secretName: datastore-v2-nemo-datastore-inline-config + - emptyDir: {} + name: temp + - name: data + persistentVolumeClaim: + claimName: datastore-shared-storage + */ + var initMode = int32(110) + var configMode = int32(420) + + volumes := []corev1.Volume{ + { + Name: "init", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreInitSecret, + DefaultMode: &initMode, + }, + }, + }, + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreConfigSecret, + DefaultMode: &initMode, + }, + }, + }, + { + Name: "inline-config-sources", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreInlineConfigSecret, + DefaultMode: &configMode, + }, + }, + }, + { + Name: "temp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: n.Spec.DataStoreParams.PVCSharedData, + }, + }, + }, + } + return volumes +} + +// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetEnvFrom() []corev1.EnvFromSource { + return []corev1.EnvFromSource{} +} + +// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetInitAppIniEnvFrom() []corev1.EnvFromSource { + return []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DataStoreSettingsSecret, + }, + }, + }, + } +} + +// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetStandardAnnotations() map[string]string { + standardAnnotations := map[string]string{ + "openshift.io/scc": "nonroot", + } + return standardAnnotations +} + +// GetNemoDatastoreV2Annotations returns annotations to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetNemoDatastoreV2Annotations() map[string]string { + standardAnnotations := n.GetStandardAnnotations() + + if n.Spec.Annotations != nil { + return utils.MergeMaps(standardAnnotations, n.Spec.Annotations) + } + + return standardAnnotations +} + +// GetServiceLabels returns merged labels to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetServiceLabels() map[string]string { + standardLabels := n.GetStandardLabels() + + if n.Spec.Labels != nil { + return utils.MergeMaps(standardLabels, n.Spec.Labels) + } + return standardLabels +} + +// GetSelectorLabels returns standard selector labels to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetSelectorLabels() map[string]string { + // TODO: add custom ones + return n.GetStandardSelectorLabels() +} + +// GetNodeSelector returns node selector labels for the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetNodeSelector() map[string]string { + return n.Spec.NodeSelector +} + +// GetTolerations returns tolerations for the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetTolerations() []corev1.Toleration { + return n.Spec.Tolerations +} + +// GetPodAffinity returns pod affinity for the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetPodAffinity() *corev1.PodAffinity { + return n.Spec.PodAffinity +} + +// GetContainerName returns name of the container for NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetContainerName() string { + return fmt.Sprintf("%s-ctr", n.Name) +} + +// GetCommand return command to override for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetCommand() []string { + return n.Spec.Command +} + +// GetArgs return arguments for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetArgs() []string { + return n.Spec.Args +} + +// GetEnv returns merged slice of standard and user specified env variables +func (n *NemoDatastoreV2) GetEnv() []corev1.EnvVar { + return utils.MergeEnvVars(n.GetStandardEnv(), n.Spec.Env) +} + +// GetImage returns container image for the NemoDatastoreV2 +func (n *NemoDatastoreV2) GetImage() string { + return fmt.Sprintf("%s:%s", n.Spec.Image.Repository, n.Spec.Image.Tag) +} + +// GetImagePullSecrets returns the image pull secrets for the NIM container +func (n *NemoDatastoreV2) GetImagePullSecrets() []string { + return n.Spec.Image.PullSecrets +} + +// GetImagePullPolicy returns the image pull policy for the NIM container +func (n *NemoDatastoreV2) GetImagePullPolicy() string { + return n.Spec.Image.PullPolicy +} + +// GetResources returns resources to allocate to the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetResources() *corev1.ResourceRequirements { + return n.Spec.Resources +} + +// GetLivenessProbe returns liveness probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetLivenessProbe() *corev1.Probe { + if n.Spec.LivenessProbe.Probe == nil { + return n.GetDefaultLivenessProbe() + } + return n.Spec.LivenessProbe.Probe +} + +// GetDefaultLivenessProbe returns the default liveness probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetDefaultLivenessProbe() *corev1.Probe { + probe := corev1.Probe{ + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health/live", + Port: intstr.IntOrString{ + Type: intstr.Type(0), + IntVal: n.Spec.Expose.Service.Port, + }, + }, + }, + } + + return &probe +} + +// GetReadinessProbe returns readiness probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetReadinessProbe() *corev1.Probe { + if n.Spec.ReadinessProbe.Probe == nil { + return n.GetDefaultReadinessProbe() + } + return n.Spec.ReadinessProbe.Probe +} + +// GetDefaultReadinessProbe returns the default readiness probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetDefaultReadinessProbe() *corev1.Probe { + probe := corev1.Probe{ + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health/ready", + Port: intstr.IntOrString{ + Type: intstr.Type(0), + IntVal: n.Spec.Expose.Service.Port, + }, + }, + }, + } + + return &probe +} + +// GetStartupProbe returns startup probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetStartupProbe() *corev1.Probe { + return n.Spec.StartupProbe.Probe +} + +// GetDefaultStartupProbe returns the default startup probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetDefaultStartupProbe() *corev1.Probe { + probe := corev1.Probe{ + InitialDelaySeconds: 40, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 180, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health/ready", + Port: intstr.IntOrString{ + Type: intstr.Type(0), + IntVal: n.Spec.Expose.Service.Port, + }, + }, + }, + } + + return &probe +} + +// GetVolumeMounts returns volumes for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + MountPath: "/tmp", + Name: "temp", + }, + { + MountPath: "/data", + Name: "data", + }, + } +} + +func (n *NemoDatastoreV2) GetVolumeMountsInitContainer() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + MountPath: "/usr/sbin", + Name: "config", + }, + { + MountPath: "/tmp", + Name: "temp", + }, + { + MountPath: "/data", + Name: "data", + }, + { + MountPath: "/env-to-ini-mounts/inlines/", + Name: "inline-config-sources", + }, + { + MountPath: "/usr/sbin/init", + Name: "init", + }, + } +} + +func (n *NemoDatastoreV2) GetInitContainers() []corev1.Container { + user := int64(1000) + return []corev1.Container{ + { + Name: "init-datastore", + Image: n.GetImage(), + ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), + Command: []string{ + "/bin/sh", "-c", + }, + Args: []string{ + "/usr/sbin/init/init_directory_structure.sh && /usr/sbin/config_environment.sh", + }, + VolumeMounts: n.GetVolumeMountsInitContainer(), + Env: n.GetInitContainerEnv(), + EnvFrom: n.GetInitAppIniEnvFrom(), + }, + { + Name: "configure-datastore", + Image: n.GetImage(), + ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), + Command: []string{ + "/bin/sh", "-c", + }, + Args: []string{ + "/usr/sbin/init/configure_gitea.sh", + }, + VolumeMounts: n.GetVolumeMountsInitContainer(), + Env: n.GetInitContainerEnv(), + EnvFrom: n.GetInitAppIniEnvFrom(), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &user, + }, + }, + } +} + +// GetServiceAccountName returns service account name for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetServiceAccountName() string { + return n.Name +} + +// GetRuntimeClass return the runtime class name for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetRuntimeClass() string { + return n.Spec.RuntimeClass +} + +// GetHPA returns the HPA spec for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetHPA() HorizontalPodAutoscalerSpec { + return n.Spec.Scale.HPA +} + +// GetServiceMonitor returns the Service Monitor details for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetServiceMonitor() ServiceMonitor { + return n.Spec.Metrics.ServiceMonitor +} + +// GetReplicas returns replicas for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetReplicas() int { + if n.IsAutoScalingEnabled() { + return 0 + } + return n.Spec.Replicas +} + +// GetDeploymentKind returns the kind of deployment for NemoDatastoreV2 +func (n *NemoDatastoreV2) GetDeploymentKind() string { + return "Deployment" +} + +// IsAutoScalingEnabled returns true if autoscaling is enabled for NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) IsAutoScalingEnabled() bool { + return n.Spec.Scale.Enabled != nil && *n.Spec.Scale.Enabled +} + +// IsIngressEnabled returns true if ingress is enabled for NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) IsIngressEnabled() bool { + return n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled +} + +// GetIngressSpec returns the Ingress spec NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetIngressSpec() networkingv1.IngressSpec { + return n.Spec.Expose.Ingress.Spec +} + +// IsServiceMonitorEnabled returns true if servicemonitor is enabled for NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) IsServiceMonitorEnabled() bool { + return n.Spec.Metrics.Enabled != nil && *n.Spec.Metrics.Enabled +} + +// GetServicePort returns the service port for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetServicePort() int32 { + return n.Spec.Expose.Service.Port +} + +// GetServiceType returns the service type for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetServiceType() string { + return string(n.Spec.Expose.Service.Type) +} + +// GetUserID returns the user ID for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetUserID() *int64 { + return n.Spec.UserID + +} + +// GetGroupID returns the group ID for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetGroupID() *int64 { + return n.Spec.GroupID + +} + +// GetServiceAccountParams return params to render ServiceAccount from templates +func (n *NemoDatastoreV2) GetServiceAccountParams() *rendertypes.ServiceAccountParams { + params := &rendertypes.ServiceAccountParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetNemoDatastoreV2Annotations() + return params +} + +// GetDeploymentParams returns params to render Deployment from templates +func (n *NemoDatastoreV2) GetDeploymentParams() *rendertypes.DeploymentParams { + params := &rendertypes.DeploymentParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetNemoDatastoreV2Annotations() + + // Set template spec + if !n.IsAutoScalingEnabled() { + params.Replicas = n.GetReplicas() + } + params.NodeSelector = n.GetNodeSelector() + params.Tolerations = n.GetTolerations() + params.Affinity = n.GetPodAffinity() + params.ImagePullSecrets = n.GetImagePullSecrets() + params.ImagePullPolicy = n.GetImagePullPolicy() + + // Set labels and selectors + params.SelectorLabels = n.GetSelectorLabels() + + // Set container spec + params.ContainerName = n.GetContainerName() + params.Env = n.GetEnv() + params.Args = n.GetArgs() + params.Command = n.GetCommand() + params.Resources = n.GetResources() + params.Image = n.GetImage() + + // Set container probes + if IsProbeEnabled(n.Spec.LivenessProbe) { + params.LivenessProbe = n.GetLivenessProbe() + } + if IsProbeEnabled(n.Spec.ReadinessProbe) { + params.ReadinessProbe = n.GetReadinessProbe() + } + if IsProbeEnabled(n.Spec.StartupProbe) { + params.StartupProbe = n.GetStartupProbe() + } + params.UserID = n.GetUserID() + params.GroupID = n.GetGroupID() + + // Set service account + params.ServiceAccountName = n.GetServiceAccountName() + + // Set runtime class + params.RuntimeClassName = n.GetRuntimeClass() + return params +} + +// GetStatefulSetParams returns params to render StatefulSet from templates +func (n *NemoDatastoreV2) GetStatefulSetParams() *rendertypes.StatefulSetParams { + + params := &rendertypes.StatefulSetParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetNemoDatastoreV2Annotations() + + // Set template spec + if !n.IsAutoScalingEnabled() { + params.Replicas = n.GetReplicas() + } + params.ServiceName = n.GetName() + params.NodeSelector = n.GetNodeSelector() + params.Tolerations = n.GetTolerations() + params.Affinity = n.GetPodAffinity() + params.ImagePullSecrets = n.GetImagePullSecrets() + params.ImagePullPolicy = n.GetImagePullPolicy() + + // Set labels and selectors + params.SelectorLabels = n.GetSelectorLabels() + + // Set container spec + params.ContainerName = n.GetContainerName() + params.Env = n.GetEnv() + params.Args = n.GetArgs() + params.Command = n.GetCommand() + params.Resources = n.GetResources() + params.Image = n.GetImage() + + // Set container probes + params.LivenessProbe = n.GetLivenessProbe() + params.ReadinessProbe = n.GetReadinessProbe() + params.StartupProbe = n.GetStartupProbe() + + // Set service account + params.ServiceAccountName = n.GetServiceAccountName() + + // Set runtime class + params.RuntimeClassName = n.GetRuntimeClass() + return params +} + +// GetServiceParams returns params to render Service from templates +func (n *NemoDatastoreV2) GetServiceParams() *rendertypes.ServiceParams { + params := &rendertypes.ServiceParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetServiceAnnotations() + + // Set service selector labels + params.SelectorLabels = n.GetSelectorLabels() + + // Set service type + params.Type = n.GetServiceType() + + // Set service ports + params.Port = n.GetServicePort() + params.PortName = "service-port" + return params +} + +// GetIngressParams returns params to render Ingress from templates +func (n *NemoDatastoreV2) GetIngressParams() *rendertypes.IngressParams { + params := &rendertypes.IngressParams{} + + params.Enabled = n.IsIngressEnabled() + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetIngressAnnotations() + params.Spec = n.GetIngressSpec() + return params +} + +// GetRoleParams returns params to render Role from templates +func (n *NemoDatastoreV2) GetRoleParams() *rendertypes.RoleParams { + params := &rendertypes.RoleParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + + // Set rules to use SCC + params.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{"nonroot"}, + Verbs: []string{"use"}, + }, + } + + return params +} + +// GetRoleBindingParams returns params to render RoleBinding from templates +func (n *NemoDatastoreV2) GetRoleBindingParams() *rendertypes.RoleBindingParams { + params := &rendertypes.RoleBindingParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + + params.ServiceAccountName = n.GetServiceAccountName() + params.RoleName = n.GetName() + return params +} + +// GetHPAParams returns params to render HPA from templates +func (n *NemoDatastoreV2) GetHPAParams() *rendertypes.HPAParams { + params := &rendertypes.HPAParams{} + + params.Enabled = n.IsAutoScalingEnabled() + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetHPAAnnotations() + + // Set HPA spec + hpa := n.GetHPA() + hpaSpec := autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + Kind: n.GetDeploymentKind(), + Name: n.GetName(), + APIVersion: "apps/v1", + }, + MinReplicas: hpa.MinReplicas, + MaxReplicas: hpa.MaxReplicas, + Metrics: hpa.Metrics, + Behavior: hpa.Behavior, + } + params.HPASpec = hpaSpec + return params +} + +// GetSCCParams return params to render SCC from templates +func (n *NemoDatastoreV2) GetSCCParams() *rendertypes.SCCParams { + params := &rendertypes.SCCParams{} + // Set metadata + params.Name = "nemo-datastore-scc" + + params.ServiceAccountName = n.GetServiceAccountName() + return params +} + +// GetServiceMonitorParams return params to render Service Monitor from templates +func (n *NemoDatastoreV2) GetServiceMonitorParams() *rendertypes.ServiceMonitorParams { + params := &rendertypes.ServiceMonitorParams{} + serviceMonitor := n.GetServiceMonitor() + params.Enabled = n.IsServiceMonitorEnabled() + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + svcLabels := n.GetServiceLabels() + maps.Copy(svcLabels, serviceMonitor.AdditionalLabels) + params.Labels = svcLabels + params.Annotations = n.GetServiceMonitorAnnotations() + + // Set Service Monitor spec + smSpec := monitoringv1.ServiceMonitorSpec{ + NamespaceSelector: monitoringv1.NamespaceSelector{MatchNames: []string{n.Namespace}}, + Selector: metav1.LabelSelector{MatchLabels: n.GetServiceLabels()}, + Endpoints: []monitoringv1.Endpoint{{Port: "service-port", ScrapeTimeout: serviceMonitor.ScrapeTimeout, Interval: serviceMonitor.Interval}}, + } + params.SMSpec = smSpec + return params +} + +func (n *NemoDatastoreV2) GetIngressAnnotations() map[string]string { + NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() + + if n.Spec.Expose.Ingress.Annotations != nil { + return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Expose.Ingress.Annotations) + } + return NemoDatastoreV2Annotations +} + +func (n *NemoDatastoreV2) GetServiceAnnotations() map[string]string { + NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() + + if n.Spec.Expose.Service.Annotations != nil { + return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Expose.Service.Annotations) + } + return NemoDatastoreV2Annotations +} + +func (n *NemoDatastoreV2) GetHPAAnnotations() map[string]string { + NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() + + if n.Spec.Scale.Annotations != nil { + return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Scale.Annotations) + } + return NemoDatastoreV2Annotations +} + +func (n *NemoDatastoreV2) GetServiceMonitorAnnotations() map[string]string { + NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() + + if n.Spec.Metrics.ServiceMonitor.Annotations != nil { + return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Metrics.ServiceMonitor.Annotations) + } + return NemoDatastoreV2Annotations +} + +func init() { + SchemeBuilder.Register(&NemoDatastoreV2{}, &NemoDatastoreV2List{}) +} diff --git a/api/apps/v1alpha1/zz_generated.deepcopy.go b/api/apps/v1alpha1/zz_generated.deepcopy.go index 0c3ce6b1..602a446d 100644 --- a/api/apps/v1alpha1/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/zz_generated.deepcopy.go @@ -1172,6 +1172,190 @@ func (in *NemoDatastoreStatus) DeepCopy() *NemoDatastoreStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2) DeepCopyInto(out *NemoDatastoreV2) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2. +func (in *NemoDatastoreV2) DeepCopy() *NemoDatastoreV2 { + if in == nil { + return nil + } + out := new(NemoDatastoreV2) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NemoDatastoreV2) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2List) DeepCopyInto(out *NemoDatastoreV2List) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NemoDatastoreV2, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2List. +func (in *NemoDatastoreV2List) DeepCopy() *NemoDatastoreV2List { + if in == nil { + return nil + } + out := new(NemoDatastoreV2List) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NemoDatastoreV2List) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2Params) DeepCopyInto(out *NemoDatastoreV2Params) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Params. +func (in *NemoDatastoreV2Params) DeepCopy() *NemoDatastoreV2Params { + if in == nil { + return nil + } + out := new(NemoDatastoreV2Params) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2Spec) DeepCopyInto(out *NemoDatastoreV2Spec) { + *out = *in + in.Image.DeepCopyInto(&out.Image) + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodAffinity != nil { + in, out := &in.PodAffinity, &out.PodAffinity + *out = new(v1.PodAffinity) + (*in).DeepCopyInto(*out) + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + in.Expose.DeepCopyInto(&out.Expose) + in.LivenessProbe.DeepCopyInto(&out.LivenessProbe) + in.ReadinessProbe.DeepCopyInto(&out.ReadinessProbe) + in.StartupProbe.DeepCopyInto(&out.StartupProbe) + in.Scale.DeepCopyInto(&out.Scale) + in.Metrics.DeepCopyInto(&out.Metrics) + if in.UserID != nil { + in, out := &in.UserID, &out.UserID + *out = new(int64) + **out = **in + } + if in.GroupID != nil { + in, out := &in.GroupID, &out.GroupID + *out = new(int64) + **out = **in + } + out.DataStoreParams = in.DataStoreParams +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Spec. +func (in *NemoDatastoreV2Spec) DeepCopy() *NemoDatastoreV2Spec { + if in == nil { + return nil + } + out := new(NemoDatastoreV2Spec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2Status) DeepCopyInto(out *NemoDatastoreV2Status) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Status. +func (in *NemoDatastoreV2Status) DeepCopy() *NemoDatastoreV2Status { + if in == nil { + return nil + } + out := new(NemoDatastoreV2Status) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NemoEntitystore) DeepCopyInto(out *NemoEntitystore) { *out = *in diff --git a/api/versioned/typed/apps/v1alpha1/apps_client.go b/api/versioned/typed/apps/v1alpha1/apps_client.go index 045f4265..b6f2e4ca 100644 --- a/api/versioned/typed/apps/v1alpha1/apps_client.go +++ b/api/versioned/typed/apps/v1alpha1/apps_client.go @@ -31,6 +31,7 @@ type AppsV1alpha1Interface interface { NIMPipelinesGetter NIMServicesGetter NemoDatastoresGetter + NemoDatastoreV2sGetter NemoEntitystoresGetter NemoEvaluatorsGetter NemoGuardrailsGetter @@ -57,6 +58,10 @@ func (c *AppsV1alpha1Client) NemoDatastores(namespace string) NemoDatastoreInter return newNemoDatastores(c, namespace) } +func (c *AppsV1alpha1Client) NemoDatastoreV2s(namespace string) NemoDatastoreV2Interface { + return newNemoDatastoreV2s(c, namespace) +} + func (c *AppsV1alpha1Client) NemoEntitystores(namespace string) NemoEntitystoreInterface { return newNemoEntitystores(c, namespace) } diff --git a/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go b/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go index 1896f204..9809e22c 100644 --- a/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go +++ b/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go @@ -43,6 +43,10 @@ func (c *FakeAppsV1alpha1) NemoDatastores(namespace string) v1alpha1.NemoDatasto return &FakeNemoDatastores{c, namespace} } +func (c *FakeAppsV1alpha1) NemoDatastoreV2s(namespace string) v1alpha1.NemoDatastoreV2Interface { + return &FakeNemoDatastoreV2s{c, namespace} +} + func (c *FakeAppsV1alpha1) NemoEntitystores(namespace string) v1alpha1.NemoEntitystoreInterface { return &FakeNemoEntitystores{c, namespace} } diff --git a/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go b/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go new file mode 100644 index 00000000..fd957d51 --- /dev/null +++ b/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go @@ -0,0 +1,146 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeNemoDatastoreV2s implements NemoDatastoreV2Interface +type FakeNemoDatastoreV2s struct { + Fake *FakeAppsV1alpha1 + ns string +} + +var nemodatastorev2sResource = v1alpha1.SchemeGroupVersion.WithResource("nemodatastorev2s") + +var nemodatastorev2sKind = v1alpha1.SchemeGroupVersion.WithKind("NemoDatastoreV2") + +// Get takes name of the nemoDatastoreV2, and returns the corresponding nemoDatastoreV2 object, and an error if there is any. +func (c *FakeNemoDatastoreV2s) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(nemodatastorev2sResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} + +// List takes label and field selectors, and returns the list of NemoDatastoreV2s that match those selectors. +func (c *FakeNemoDatastoreV2s) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.NemoDatastoreV2List, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2List{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(nemodatastorev2sResource, nemodatastorev2sKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.NemoDatastoreV2List{ListMeta: obj.(*v1alpha1.NemoDatastoreV2List).ListMeta} + for _, item := range obj.(*v1alpha1.NemoDatastoreV2List).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested nemoDatastoreV2s. +func (c *FakeNemoDatastoreV2s) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(nemodatastorev2sResource, c.ns, opts)) + +} + +// Create takes the representation of a nemoDatastoreV2 and creates it. Returns the server's representation of the nemoDatastoreV2, and an error, if there is any. +func (c *FakeNemoDatastoreV2s) Create(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.CreateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(nemodatastorev2sResource, c.ns, nemoDatastoreV2, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} + +// Update takes the representation of a nemoDatastoreV2 and updates it. Returns the server's representation of the nemoDatastoreV2, and an error, if there is any. +func (c *FakeNemoDatastoreV2s) Update(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(nemodatastorev2sResource, c.ns, nemoDatastoreV2, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeNemoDatastoreV2s) UpdateStatus(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(nemodatastorev2sResource, "status", c.ns, nemoDatastoreV2, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} + +// Delete takes name of the nemoDatastoreV2 and deletes it. Returns an error if one occurs. +func (c *FakeNemoDatastoreV2s) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(nemodatastorev2sResource, c.ns, name, opts), &v1alpha1.NemoDatastoreV2{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeNemoDatastoreV2s) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(nemodatastorev2sResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.NemoDatastoreV2List{}) + return err +} + +// Patch applies the patch and returns the patched nemoDatastoreV2. +func (c *FakeNemoDatastoreV2s) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(nemodatastorev2sResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} diff --git a/api/versioned/typed/apps/v1alpha1/generated_expansion.go b/api/versioned/typed/apps/v1alpha1/generated_expansion.go index 78164a8c..59602980 100644 --- a/api/versioned/typed/apps/v1alpha1/generated_expansion.go +++ b/api/versioned/typed/apps/v1alpha1/generated_expansion.go @@ -25,6 +25,8 @@ type NIMServiceExpansion interface{} type NemoDatastoreExpansion interface{} +type NemoDatastoreV2Expansion interface{} + type NemoEntitystoreExpansion interface{} type NemoEvaluatorExpansion interface{} diff --git a/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go b/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go new file mode 100644 index 00000000..d58def11 --- /dev/null +++ b/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go @@ -0,0 +1,68 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" + scheme "github.com/NVIDIA/k8s-nim-operator/api/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// NemoDatastoreV2sGetter has a method to return a NemoDatastoreV2Interface. +// A group's client should implement this interface. +type NemoDatastoreV2sGetter interface { + NemoDatastoreV2s(namespace string) NemoDatastoreV2Interface +} + +// NemoDatastoreV2Interface has methods to work with NemoDatastoreV2 resources. +type NemoDatastoreV2Interface interface { + Create(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.CreateOptions) (*v1alpha1.NemoDatastoreV2, error) + Update(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (*v1alpha1.NemoDatastoreV2, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (*v1alpha1.NemoDatastoreV2, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.NemoDatastoreV2, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.NemoDatastoreV2List, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NemoDatastoreV2, err error) + NemoDatastoreV2Expansion +} + +// nemoDatastoreV2s implements NemoDatastoreV2Interface +type nemoDatastoreV2s struct { + *gentype.ClientWithList[*v1alpha1.NemoDatastoreV2, *v1alpha1.NemoDatastoreV2List] +} + +// newNemoDatastoreV2s returns a NemoDatastoreV2s +func newNemoDatastoreV2s(c *AppsV1alpha1Client, namespace string) *nemoDatastoreV2s { + return &nemoDatastoreV2s{ + gentype.NewClientWithList[*v1alpha1.NemoDatastoreV2, *v1alpha1.NemoDatastoreV2List]( + "nemodatastorev2s", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.NemoDatastoreV2 { return &v1alpha1.NemoDatastoreV2{} }, + func() *v1alpha1.NemoDatastoreV2List { return &v1alpha1.NemoDatastoreV2List{} }), + } +} diff --git a/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml b/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml new file mode 100644 index 00000000..d72012e9 --- /dev/null +++ b/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml @@ -0,0 +1,2212 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nemodatastorev2s.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NemoDatastoreV2 + listKind: NemoDatastoreV2List + plural: nemodatastorev2s + singular: nemodatastorev2 + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 + properties: + annotations: + additionalProperties: + type: string + type: object + args: + items: + type: string + type: array + authSecret: + description: The name of an secret that contains authn for the NGC + NIM service API + type: string + command: + items: + type: string + type: array + dataStoreParams: + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + datastoreSettingsSecret: + type: string + dbSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + objStoreSecret: + type: string + pvcSharedData: + type: string + size: + type: string + sshEnabled: + type: boolean + storageClass: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret + - dbSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - pvcSharedData + - sshEnabled + - storageClass + type: object + env: + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + expose: + description: Expose defines attributes to expose the service + properties: + ingress: + description: Ingress defines attributes to enable ingress for + the service + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + description: ingress, or virtualService - not both + type: boolean + spec: + description: IngressSpec describes the Ingress the user wishes + to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain + name of a network host, as defined by RFC 3986.\nNote + the following deviations from the \"host\" part + of the\nURI as defined in RFC 3986:\n1. IPs are + not allowed. Currently an IngressRuleValue can + only apply to\n the IP in the Spec of the parent + Ingress.\n2. The `:` delimiter is not respected + because ports are not allowed.\n\t Currently + the port of an Ingress is implicitly :80 for http + and\n\t :443 for https.\nBoth these may change + in the future.\nIncoming requests are matched + against the host before the\nIngressRuleValue. + If the host is unspecified, the Ingress routes + all\ntraffic based on the specified IngressRuleValue.\n\nhost + can be \"precise\" which is a domain name without + the terminating dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by itself as + the first DNS label and\nmatches only a single + label. You cannot have a wildcard label by itself + (e.g. Host == \"*\").\nRequests will be matched + against the Host field in the following way:\n1. + If host is precise, the request matches this rule + if the http host header is equal to Host.\n2. + If host is a wildcard, then the request matches + this rule if the http host header\nis to equal + to the suffix (removing the first label) of the + wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths + that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer + security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + service: + description: Service defines attributes to create a service + properties: + annotations: + additionalProperties: + type: string + type: object + name: + description: override the default service name + type: string + port: + default: 8000 + format: int32 + type: integer + type: + description: Service Type string describes ingress methods + for a service + type: string + required: + - port + type: object + type: object + groupID: + format: int64 + type: integer + image: + description: Image defines image attributes + properties: + pullPolicy: + type: string + pullSecrets: + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + metrics: + description: Metrics defines attributes to setup metrics collection + properties: + enabled: + type: boolean + serviceMonitor: + description: for use with the Prometheus Operator and the primary + service object + properties: + additionalLabels: + additionalProperties: + type: string + type: object + annotations: + additionalProperties: + type: string + type: object + interval: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + scrapeTimeout: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with + the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + readinessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + replicas: + default: 1 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + runtimeClass: + type: string + scale: + description: Autoscaling defines attributes to automatically scale + the service based on metrics + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + hpa: + description: HorizontalPodAutoscalerSpec defines the parameters + required to setup HPA + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of + the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + Note: "ContainerResource" type is available on when the feature-gate + HPAContainerMetrics is enabled + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + startupProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + format: int64 + type: integer + required: + - authSecret + - dataStoreParams + type: object + status: + description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 + properties: + availableReplicas: + format: int32 + type: integer + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/cmd/main.go b/cmd/main.go index bd84de2a..d0fb4d33 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -221,6 +221,17 @@ func main() { os.Exit(1) } + if err = controller.NewNemoDatastoreV2Reconciler( + mgr.GetClient(), + mgr.GetScheme(), + updater, + render.NewRenderer("/manifests"), + ctrl.Log.WithName("controllers").WithName("NemoDatastoreV2"), + ).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NemoDatastoreV2") + os.Exit(1) + } + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml b/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml new file mode 100644 index 00000000..d72012e9 --- /dev/null +++ b/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml @@ -0,0 +1,2212 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nemodatastorev2s.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NemoDatastoreV2 + listKind: NemoDatastoreV2List + plural: nemodatastorev2s + singular: nemodatastorev2 + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 + properties: + annotations: + additionalProperties: + type: string + type: object + args: + items: + type: string + type: array + authSecret: + description: The name of an secret that contains authn for the NGC + NIM service API + type: string + command: + items: + type: string + type: array + dataStoreParams: + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + datastoreSettingsSecret: + type: string + dbSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + objStoreSecret: + type: string + pvcSharedData: + type: string + size: + type: string + sshEnabled: + type: boolean + storageClass: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret + - dbSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - pvcSharedData + - sshEnabled + - storageClass + type: object + env: + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + expose: + description: Expose defines attributes to expose the service + properties: + ingress: + description: Ingress defines attributes to enable ingress for + the service + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + description: ingress, or virtualService - not both + type: boolean + spec: + description: IngressSpec describes the Ingress the user wishes + to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain + name of a network host, as defined by RFC 3986.\nNote + the following deviations from the \"host\" part + of the\nURI as defined in RFC 3986:\n1. IPs are + not allowed. Currently an IngressRuleValue can + only apply to\n the IP in the Spec of the parent + Ingress.\n2. The `:` delimiter is not respected + because ports are not allowed.\n\t Currently + the port of an Ingress is implicitly :80 for http + and\n\t :443 for https.\nBoth these may change + in the future.\nIncoming requests are matched + against the host before the\nIngressRuleValue. + If the host is unspecified, the Ingress routes + all\ntraffic based on the specified IngressRuleValue.\n\nhost + can be \"precise\" which is a domain name without + the terminating dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by itself as + the first DNS label and\nmatches only a single + label. You cannot have a wildcard label by itself + (e.g. Host == \"*\").\nRequests will be matched + against the Host field in the following way:\n1. + If host is precise, the request matches this rule + if the http host header is equal to Host.\n2. + If host is a wildcard, then the request matches + this rule if the http host header\nis to equal + to the suffix (removing the first label) of the + wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths + that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer + security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + service: + description: Service defines attributes to create a service + properties: + annotations: + additionalProperties: + type: string + type: object + name: + description: override the default service name + type: string + port: + default: 8000 + format: int32 + type: integer + type: + description: Service Type string describes ingress methods + for a service + type: string + required: + - port + type: object + type: object + groupID: + format: int64 + type: integer + image: + description: Image defines image attributes + properties: + pullPolicy: + type: string + pullSecrets: + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + metrics: + description: Metrics defines attributes to setup metrics collection + properties: + enabled: + type: boolean + serviceMonitor: + description: for use with the Prometheus Operator and the primary + service object + properties: + additionalLabels: + additionalProperties: + type: string + type: object + annotations: + additionalProperties: + type: string + type: object + interval: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + scrapeTimeout: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with + the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + readinessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + replicas: + default: 1 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + runtimeClass: + type: string + scale: + description: Autoscaling defines attributes to automatically scale + the service based on metrics + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + hpa: + description: HorizontalPodAutoscalerSpec defines the parameters + required to setup HPA + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of + the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + Note: "ContainerResource" type is available on when the feature-gate + HPAContainerMetrics is enabled + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + startupProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + format: int64 + type: integer + required: + - authSecret + - dataStoreParams + type: object + status: + description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 + properties: + availableReplicas: + format: int32 + type: integer + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4a01d8f8..1a736c2c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -63,6 +63,7 @@ rules: - apps.nvidia.com resources: - nemodatastores + - nemodatastorev2s - nemoentitystores - nemoevaluators - nemoguardrails @@ -81,6 +82,7 @@ rules: - apps.nvidia.com resources: - nemodatastores/finalizers + - nemodatastorev2s/finalizers - nemoentitystores/finalizers - nemoevaluators/finalizers - nemoguardrails/finalizers @@ -93,6 +95,7 @@ rules: - apps.nvidia.com resources: - nemodatastores/status + - nemodatastorev2s/status - nemoentitystores/status - nemoevaluators/status - nemoguardrails/status diff --git a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml new file mode 100644 index 00000000..d72012e9 --- /dev/null +++ b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml @@ -0,0 +1,2212 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nemodatastorev2s.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NemoDatastoreV2 + listKind: NemoDatastoreV2List + plural: nemodatastorev2s + singular: nemodatastorev2 + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 + properties: + annotations: + additionalProperties: + type: string + type: object + args: + items: + type: string + type: array + authSecret: + description: The name of an secret that contains authn for the NGC + NIM service API + type: string + command: + items: + type: string + type: array + dataStoreParams: + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + datastoreSettingsSecret: + type: string + dbSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + objStoreSecret: + type: string + pvcSharedData: + type: string + size: + type: string + sshEnabled: + type: boolean + storageClass: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret + - dbSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - pvcSharedData + - sshEnabled + - storageClass + type: object + env: + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + expose: + description: Expose defines attributes to expose the service + properties: + ingress: + description: Ingress defines attributes to enable ingress for + the service + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + description: ingress, or virtualService - not both + type: boolean + spec: + description: IngressSpec describes the Ingress the user wishes + to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain + name of a network host, as defined by RFC 3986.\nNote + the following deviations from the \"host\" part + of the\nURI as defined in RFC 3986:\n1. IPs are + not allowed. Currently an IngressRuleValue can + only apply to\n the IP in the Spec of the parent + Ingress.\n2. The `:` delimiter is not respected + because ports are not allowed.\n\t Currently + the port of an Ingress is implicitly :80 for http + and\n\t :443 for https.\nBoth these may change + in the future.\nIncoming requests are matched + against the host before the\nIngressRuleValue. + If the host is unspecified, the Ingress routes + all\ntraffic based on the specified IngressRuleValue.\n\nhost + can be \"precise\" which is a domain name without + the terminating dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by itself as + the first DNS label and\nmatches only a single + label. You cannot have a wildcard label by itself + (e.g. Host == \"*\").\nRequests will be matched + against the Host field in the following way:\n1. + If host is precise, the request matches this rule + if the http host header is equal to Host.\n2. + If host is a wildcard, then the request matches + this rule if the http host header\nis to equal + to the suffix (removing the first label) of the + wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths + that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer + security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + service: + description: Service defines attributes to create a service + properties: + annotations: + additionalProperties: + type: string + type: object + name: + description: override the default service name + type: string + port: + default: 8000 + format: int32 + type: integer + type: + description: Service Type string describes ingress methods + for a service + type: string + required: + - port + type: object + type: object + groupID: + format: int64 + type: integer + image: + description: Image defines image attributes + properties: + pullPolicy: + type: string + pullSecrets: + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + metrics: + description: Metrics defines attributes to setup metrics collection + properties: + enabled: + type: boolean + serviceMonitor: + description: for use with the Prometheus Operator and the primary + service object + properties: + additionalLabels: + additionalProperties: + type: string + type: object + annotations: + additionalProperties: + type: string + type: object + interval: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + scrapeTimeout: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with + the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + readinessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + replicas: + default: 1 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + runtimeClass: + type: string + scale: + description: Autoscaling defines attributes to automatically scale + the service based on metrics + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + hpa: + description: HorizontalPodAutoscalerSpec defines the parameters + required to setup HPA + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of + the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + Note: "ContainerResource" type is available on when the feature-gate + HPAContainerMetrics is enabled + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + startupProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + format: int64 + type: integer + required: + - authSecret + - dataStoreParams + type: object + status: + description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 + properties: + availableReplicas: + format: int32 + type: integer + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/internal/controller/nemo_datastorev2_controller.go b/internal/controller/nemo_datastorev2_controller.go new file mode 100644 index 00000000..3f2ead39 --- /dev/null +++ b/internal/controller/nemo_datastorev2_controller.go @@ -0,0 +1,642 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "reflect" + + "k8s.io/apimachinery/pkg/api/resource" + + appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" + "github.com/NVIDIA/k8s-nim-operator/internal/conditions" + "github.com/NVIDIA/k8s-nim-operator/internal/k8sutil" + "github.com/NVIDIA/k8s-nim-operator/internal/render" + "github.com/NVIDIA/k8s-nim-operator/internal/shared" + "github.com/NVIDIA/k8s-nim-operator/internal/utils" + "github.com/go-logr/logr" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// NemoDatastoreV2Finalizer is the finalizer annotation +const NemoDatastoreV2Finalizer = "finalizer.NemoDatastoreV2.apps.nvidia.com" + +// NemoDatastoreV2Reconciler reconciles a NemoDatastoreV2 object +type NemoDatastoreV2Reconciler struct { + client.Client + scheme *runtime.Scheme + log logr.Logger + updater conditions.Updater + renderer render.Renderer + Config *rest.Config + recorder record.EventRecorder + orchestratorType k8sutil.OrchestratorType +} + +// Ensure NemoDatastoreV2Reconciler implements the Reconciler interface +var _ shared.Reconciler = &NemoDatastoreV2Reconciler{} + +// NewNemoDatastoreV2Reconciler creates a new reconciler for NemoDatastoreV2 with the given platform +func NewNemoDatastoreV2Reconciler(client client.Client, scheme *runtime.Scheme, updater conditions.Updater, renderer render.Renderer, log logr.Logger) *NemoDatastoreV2Reconciler { + return &NemoDatastoreV2Reconciler{ + Client: client, + scheme: scheme, + updater: updater, + renderer: renderer, + log: log, + } +} + +// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s/finalizers,verbs=update +// +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions;proxies,verbs=get;list;watch +// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=use,resourceNames=nonroot +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=serviceaccounts;pods;pods/eviction;services;services/finalizers;endpoints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims;configmaps;secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalars,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the NemoDatastoreV2 object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.2/pkg/reconcile +func (r *NemoDatastoreV2Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + defer r.refreshMetrics(ctx) + + // Fetch the NemoDatastoreV2 instance + NemoDatastoreV2 := &appsv1alpha1.NemoDatastoreV2{} + if err := r.Get(ctx, req.NamespacedName, NemoDatastoreV2); err != nil { + if client.IgnoreNotFound(err) != nil { + logger.Error(err, "unable to fetch NemoDatastoreV2", "name", req.NamespacedName) + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + logger.Info("Reconciling", "NemoDatastoreV2", NemoDatastoreV2.Name) + + // Check if the instance is marked for deletion + if NemoDatastoreV2.ObjectMeta.DeletionTimestamp.IsZero() { + // Add finalizer if not present + if !controllerutil.ContainsFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) { + controllerutil.AddFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) + if err := r.Update(ctx, NemoDatastoreV2); err != nil { + return ctrl.Result{}, err + } + } + } else { + // The instance is being deleted + if controllerutil.ContainsFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) { + // Perform platform specific cleanup of resources + if err := r.cleanupNemoDatastoreV2(ctx, NemoDatastoreV2); err != nil { + r.GetEventRecorder().Eventf(NemoDatastoreV2, corev1.EventTypeNormal, "Delete", + "NemoDatastoreV2 %s in deleted", NemoDatastoreV2.Name) + return ctrl.Result{}, err + } + + // Remove finalizer to allow for deletion + controllerutil.RemoveFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) + if err := r.Update(ctx, NemoDatastoreV2); err != nil { + r.GetEventRecorder().Eventf(NemoDatastoreV2, corev1.EventTypeNormal, "Delete", + "NemoDatastoreV2 %s finalizer removed", NemoDatastoreV2.Name) + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + } + + // Fetch container orchestrator type + _, err := r.GetOrchestratorType() + if err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to get container orchestrator type, %v", err) + } + + // Handle platform-specific reconciliation + if result, err := r.reconcileNemoDatastoreV2(ctx, NemoDatastoreV2); err != nil { + logger.Error(err, "error reconciling NemoDatastoreV2", "name", NemoDatastoreV2.Name) + return result, err + } + + return ctrl.Result{}, nil +} + +func (r *NemoDatastoreV2Reconciler) cleanupNemoDatastoreV2(ctx context.Context, nimService *appsv1alpha1.NemoDatastoreV2) error { + // All dependent (owned) objects will be automatically garbage collected. + // TODO: Handle any custom cleanup logic for the NIM microservice + return nil +} + +// GetScheme returns the scheme of the reconciler +func (r *NemoDatastoreV2Reconciler) GetScheme() *runtime.Scheme { + return r.scheme +} + +// GetLogger returns the logger of the reconciler +func (r *NemoDatastoreV2Reconciler) GetLogger() logr.Logger { + return r.log +} + +// GetClient returns the client instance +func (r *NemoDatastoreV2Reconciler) GetClient() client.Client { + return r.Client +} + +// GetConfig returns the rest config +func (r *NemoDatastoreV2Reconciler) GetConfig() *rest.Config { + return r.Config +} + +// GetUpdater returns the conditions updater instance +func (r *NemoDatastoreV2Reconciler) GetUpdater() conditions.Updater { + return r.updater +} + +// GetRenderer returns the renderer instance +func (r *NemoDatastoreV2Reconciler) GetRenderer() render.Renderer { + return r.renderer +} + +// GetEventRecorder returns the event recorder +func (r *NemoDatastoreV2Reconciler) GetEventRecorder() record.EventRecorder { + return r.recorder +} + +// GetOrchestratorType returns the container platform type +func (r *NemoDatastoreV2Reconciler) GetOrchestratorType() (k8sutil.OrchestratorType, error) { + if r.orchestratorType == "" { + orchestratorType, err := k8sutil.GetOrchestratorType(r.GetClient()) + if err != nil { + return k8sutil.Unknown, fmt.Errorf("Unable to get container orchestrator type, %v", err) + } + r.orchestratorType = orchestratorType + r.GetLogger().Info("Container orchestrator is successfully set", "type", orchestratorType) + } + return r.orchestratorType, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NemoDatastoreV2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.recorder = mgr.GetEventRecorderFor("nemo-datastore-service-controller") + return ctrl.NewControllerManagedBy(mgr). + For(&appsv1alpha1.NemoDatastoreV2{}). + Owns(&appsv1.Deployment{}). + Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Service{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + Owns(&networkingv1.Ingress{}). + Owns(&autoscalingv2.HorizontalPodAutoscaler{}). + WithEventFilter(predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + // Type assert to NemoDatastoreV2 + if oldNemoDatastoreV2, ok := e.ObjectOld.(*appsv1alpha1.NemoDatastoreV2); ok { + newNemoDatastoreV2 := e.ObjectNew.(*appsv1alpha1.NemoDatastoreV2) + + // Handle case where object is marked for deletion + if !newNemoDatastoreV2.ObjectMeta.DeletionTimestamp.IsZero() { + return true + } + + // Handle only spec updates + return !reflect.DeepEqual(oldNemoDatastoreV2.Spec, newNemoDatastoreV2.Spec) + } + // For other types we watch, reconcile them + return true + }, + }). + Complete(r) +} + +func (r *NemoDatastoreV2Reconciler) refreshMetrics(ctx context.Context) { + logger := log.FromContext(ctx) + // List all nodes + NemoDatastoreV2List := &appsv1alpha1.NemoDatastoreV2List{} + err := r.Client.List(ctx, NemoDatastoreV2List, &client.ListOptions{}) + if err != nil { + logger.Error(err, "unable to list NemoDatastoreV2s in the cluster") + return + } + //refreshNemoDatastoreV2Metrics(NemoDatastoreV2List) +} + +func (r *NemoDatastoreV2Reconciler) reconcileNemoDatastoreV2(ctx context.Context, nemoDatastore *appsv1alpha1.NemoDatastoreV2) (ctrl.Result, error) { + logger := log.FromContext(ctx) + var err error + defer func() { + if err != nil { + r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeWarning, conditions.Failed, + "NemoDatastoreV2 %s failed, msg: %s", nemoDatastore.Name, err.Error()) + } + }() + + pvc := corev1.PersistentVolumeClaim{} + pvc.Name = nemoDatastore.Spec.DataStoreParams.PVCSharedData + pvc.Spec.StorageClassName = &nemoDatastore.Spec.DataStoreParams.StorageClass + size, err := resource.ParseQuantity(nemoDatastore.Spec.DataStoreParams.Size) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to parse size for pvc creation %s, err %v", nemoDatastore.Name, err) + } + pvc.Spec.Resources = corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: size, + }, + } + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pvc.Namespace = nemoDatastore.Namespace + + err = r.Create(ctx, &pvc) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "Failed to create pvc", "name", pvc.Name) + } + + // Generate annotation for the current operator-version and apply to all resources + // Get generic name for all resources + namespacedName := types.NamespacedName{Name: nemoDatastore.GetName(), Namespace: nemoDatastore.GetNamespace()} + + renderer := r.GetRenderer() + + // Sync serviceaccount + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &corev1.ServiceAccount{}, func() (client.Object, error) { + return renderer.ServiceAccount(nemoDatastore.GetServiceAccountParams()) + }, "serviceaccount", conditions.ReasonServiceAccountFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Sync role + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &rbacv1.Role{}, func() (client.Object, error) { + return renderer.Role(nemoDatastore.GetRoleParams()) + }, "role", conditions.ReasonRoleFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Sync rolebinding + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &rbacv1.RoleBinding{}, func() (client.Object, error) { + return renderer.RoleBinding(nemoDatastore.GetRoleBindingParams()) + }, "rolebinding", conditions.ReasonRoleBindingFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Sync service + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &corev1.Service{}, func() (client.Object, error) { + return renderer.Service(nemoDatastore.GetServiceParams()) + }, "service", conditions.ReasonServiceFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Sync ingress + if nemoDatastore.IsIngressEnabled() { + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &networkingv1.Ingress{}, func() (client.Object, error) { + return renderer.Ingress(nemoDatastore.GetIngressParams()) + }, "ingress", conditions.ReasonIngressFailed) + if err != nil { + return ctrl.Result{}, err + } + } else { + err = r.cleanupResource(ctx, &networkingv1.Ingress{}, namespacedName) + if err != nil && !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + // Sync HPA + if nemoDatastore.IsAutoScalingEnabled() { + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &autoscalingv2.HorizontalPodAutoscaler{}, func() (client.Object, error) { + return renderer.HPA(nemoDatastore.GetHPAParams()) + }, "hpa", conditions.ReasonHPAFailed) + if err != nil { + return ctrl.Result{}, err + } + } else { + // If autoscaling is disabled, ensure the HPA is deleted + err = r.cleanupResource(ctx, &autoscalingv2.HorizontalPodAutoscaler{}, namespacedName) + if err != nil { + return ctrl.Result{}, err + } + } + + // Sync Service Monitor + if nemoDatastore.IsServiceMonitorEnabled() { + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &monitoringv1.ServiceMonitor{}, func() (client.Object, error) { + return renderer.ServiceMonitor(nemoDatastore.GetServiceMonitorParams()) + }, "servicemonitor", conditions.ReasonServiceMonitorFailed) + if err != nil { + return ctrl.Result{}, err + } + } + + deploymentParams := nemoDatastore.GetDeploymentParams() + + // Setup volume mounts with model store + deploymentParams.Volumes = nemoDatastore.GetVolumes() + deploymentParams.VolumeMounts = nemoDatastore.GetVolumeMounts() + + logger.Info("Reconciling", "volumes", nemoDatastore.GetVolumes()) + + // Sync deployment + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &appsv1.Deployment{}, func() (client.Object, error) { + result, err := renderer.Deployment(deploymentParams) + if err != nil { + return nil, err + } + initContainers := nemoDatastore.GetInitContainers() + if len(initContainers) > 0 { + result.Spec.Template.Spec.InitContainers = initContainers + } + envFrom := nemoDatastore.GetEnvFrom() + if len(envFrom) > 0 { + result.Spec.Template.Spec.Containers[0].EnvFrom = envFrom + } + fsGroup := int64(1000) + result.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ + FSGroup: &fsGroup, + } + return result, nil + }, "deployment", conditions.ReasonDeploymentFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Wait for deployment + msg, ready, err := r.isDeploymentReady(ctx, &namespacedName) + if err != nil { + return ctrl.Result{}, err + } + + if !ready { + // Update status as NotReady + err = r.SetConditionsNotReady(ctx, nemoDatastore, conditions.NotReady, msg) + r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeNormal, conditions.NotReady, + "NemoDatastoreV2 %s not ready yet, msg: %s", nemoDatastore.Name, msg) + } else { + // Update status as ready + err = r.SetConditionsReady(ctx, nemoDatastore, conditions.Ready, msg) + r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeNormal, conditions.Ready, + "NemoDatastoreV2 %s ready, msg: %s", nemoDatastore.Name, msg) + } + + if err != nil { + logger.Error(err, "Unable to update status") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *NemoDatastoreV2Reconciler) renderAndSyncResource(ctx context.Context, NemoDatastoreV2 client.Object, renderer *render.Renderer, obj client.Object, renderFunc func() (client.Object, error), conditionType string, reason string) error { + logger := log.FromContext(ctx) + + namespacedName := types.NamespacedName{Name: NemoDatastoreV2.GetName(), Namespace: NemoDatastoreV2.GetNamespace()} + + resource, err := renderFunc() + if err != nil { + logger.Error(err, "failed to render", conditionType, namespacedName) + statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) + if statusError != nil { + logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) + } + return err + } + + // Check if the resource is nil + if resource == nil { + logger.V(2).Info("rendered nil resource") + return nil + } + + metaAccessor, ok := resource.(metav1.Object) + if !ok { + logger.V(2).Info("rendered un-initialized resource") + return nil + } + + if metaAccessor == nil || metaAccessor.GetName() == "" || metaAccessor.GetNamespace() == "" { + logger.V(2).Info("rendered un-initialized resource") + return nil + } + + if err = controllerutil.SetControllerReference(NemoDatastoreV2, resource, r.GetScheme()); err != nil { + logger.Error(err, "failed to set owner", conditionType, namespacedName) + statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) + if statusError != nil { + logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) + } + return err + } + + err = r.syncResource(ctx, obj, resource, namespacedName) + if err != nil { + logger.Error(err, "failed to sync", conditionType, namespacedName) + statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) + if statusError != nil { + logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) + } + return err + } + return nil +} + +// CheckDeploymentReadiness checks if the Deployment is ready +func (r *NemoDatastoreV2Reconciler) isDeploymentReady(ctx context.Context, namespacedName *types.NamespacedName) (string, bool, error) { + deployment := &appsv1.Deployment{} + err := r.Get(ctx, client.ObjectKey{Name: namespacedName.Name, Namespace: namespacedName.Namespace}, deployment) + if err != nil { + if errors.IsNotFound(err) { + return "", false, nil + } + return "", false, err + } + + cond := getDeploymentCondition(deployment.Status, appsv1.DeploymentProgressing) + if cond != nil && cond.Reason == "ProgressDeadlineExceeded" { + return fmt.Sprintf("deployment %q exceeded its progress deadline", deployment.Name), false, nil + } + if deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d out of %d new replicas have been updated...\n", deployment.Name, deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas), false, nil + } + if deployment.Status.Replicas > deployment.Status.UpdatedReplicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d old replicas are pending termination...\n", deployment.Name, deployment.Status.Replicas-deployment.Status.UpdatedReplicas), false, nil + } + if deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d of %d updated replicas are available...\n", deployment.Name, deployment.Status.AvailableReplicas, deployment.Status.UpdatedReplicas), false, nil + } + return fmt.Sprintf("deployment %q successfully rolled out\n", deployment.Name), true, nil +} + +func (r *NemoDatastoreV2Reconciler) syncResource(ctx context.Context, obj client.Object, desired client.Object, namespacedName types.NamespacedName) error { + logger := log.FromContext(ctx) + + err := r.Get(ctx, namespacedName, obj) + if err != nil && !errors.IsNotFound(err) { + return err + } + + if !utils.IsSpecChanged(obj, desired) { + logger.V(2).Info("Object spec has not changed, skipping update", "obj", obj) + return nil + } + logger.V(2).Info("Object spec has changed, updating") + + if errors.IsNotFound(err) { + err = r.Create(ctx, desired) + if err != nil { + return err + } + } else { + err = r.Update(ctx, desired) + if err != nil { + return err + } + } + return nil +} + +// cleanupResource deletes the given Kubernetes resource if it exists. +// If the resource does not exist or an error occurs during deletion, the function returns nil or the error. +// +// Parameters: +// ctx (context.Context): The context for the operation. +// obj (client.Object): The Kubernetes resource to delete. +// namespacedName (types.NamespacedName): The namespaced name of the resource. +// +// Returns: +// error: An error if the resource deletion fails, or nil if the resource is not found or deletion is successful. +func (r *NemoDatastoreV2Reconciler) cleanupResource(ctx context.Context, obj client.Object, namespacedName types.NamespacedName) error { + + logger := log.FromContext(ctx) + + err := r.Get(ctx, namespacedName, obj) + if err != nil && !errors.IsNotFound(err) { + return err + } + + if errors.IsNotFound(err) { + return nil + } + + err = r.Delete(ctx, obj) + if err != nil { + return err + } + logger.V(2).Info("NIM Service object changed, deleting ", "obj", obj) + return nil +} + +func (r *NemoDatastoreV2Reconciler) SetConditionsReady(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Ready, + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + }) + + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Failed, + Status: metav1.ConditionFalse, + Reason: conditions.Ready, + }) + + cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusReady + return r.updateNemoDatastoreV2Status(ctx, cr) +} + +func (r *NemoDatastoreV2Reconciler) SetConditionsNotReady(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Ready, + Status: metav1.ConditionFalse, + Reason: reason, + Message: message, + }) + + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Failed, + Status: metav1.ConditionFalse, + Reason: conditions.Ready, + Message: message, + }) + + cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusNotReady + return r.updateNemoDatastoreV2Status(ctx, cr) +} + +func (r *NemoDatastoreV2Reconciler) SetConditionsFailed(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Ready, + Status: metav1.ConditionFalse, + Reason: conditions.Failed, + }) + + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Failed, + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + }) + cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusFailed + return r.updateNemoDatastoreV2Status(ctx, cr) +} + +func (r *NemoDatastoreV2Reconciler) updateNemoDatastoreV2Status(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2) error { + + obj := &appsv1alpha1.NemoDatastoreV2{} + errGet := r.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.GetNamespace()}, obj) + if errGet != nil { + return errGet + } + obj.Status = cr.Status + if err := r.Status().Update(ctx, obj); err != nil { + return err + } + return nil +} diff --git a/manifests/deployment.yaml b/manifests/deployment.yaml index cd22d174..da2c28d5 100644 --- a/manifests/deployment.yaml +++ b/manifests/deployment.yaml @@ -78,6 +78,11 @@ spec: configMap: name: {{ .ConfigMap.Name }} {{- end }} + {{- if .Secret }} + secret: + secretName: {{ .Secret.SecretName }} + defaultMode: {{ .Secret.DefaultMode }} + {{- end }} {{- if .EmptyDir }} emptyDir: medium: {{ .EmptyDir.Medium }} From d4009d8947378424152033cdf5828686fe517567 Mon Sep 17 00:00:00 2001 From: Sheng Lin Date: Sat, 18 Jan 2025 18:18:05 -0500 Subject: [PATCH 2/4] Support DataStore V2 Signed-off-by: Sheng Lin --- api/apps/v1alpha1/nemo_datastorev2_types.go | 161 ++++++++++++++++-- api/apps/v1alpha1/zz_generated.deepcopy.go | 39 ++++- .../apps.nvidia.com_nemodatastorev2s.yaml | 34 +++- .../apps.nvidia.com_nemodatastorev2s.yaml | 34 +++- .../samples/apps_v1alpha1_nimdatastorev2.yaml | 65 +++++++ .../apps.nvidia.com_nemodatastorev2s.yaml | 34 +++- .../controller/nemo_datastorev2_controller.go | 31 ++-- 7 files changed, 337 insertions(+), 61 deletions(-) create mode 100644 config/samples/apps_v1alpha1_nimdatastorev2.yaml diff --git a/api/apps/v1alpha1/nemo_datastorev2_types.go b/api/apps/v1alpha1/nemo_datastorev2_types.go index c32aad68..cddac0a2 100644 --- a/api/apps/v1alpha1/nemo_datastorev2_types.go +++ b/api/apps/v1alpha1/nemo_datastorev2_types.go @@ -28,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -102,9 +103,17 @@ type NemoDatastoreV2Params struct { SshEnabled bool `json:"sshEnabled"` - PVCSharedData string `json:"pvcSharedData"` - StorageClass string `json:"storageClass"` - Size string `json:"size,omitempty"` + Persistence *Persistence `json:"persistence,omitempty"` +} + +type Persistence struct { + Create bool `json:"create"` + Enabled bool `json:"enabled"` + ClaimName string `json:"claimName"` + Size string `json:"size"` + AccessModes []corev1.PersistentVolumeAccessMode `json:"accessModes,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + StorageClass *string `json:"storageClass,omitempty"` } // +genclient @@ -196,11 +205,44 @@ func (n *NemoDatastoreV2) GetStandardEnv() []corev1.EnvVar { Name: "HOME", Value: "/data/gitea/git", }, + { + Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "objectStoreKey", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.ObjectStoreSecret, + }, + }, + }, + }, + { + Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "objectStoreSecret", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.ObjectStoreSecret, + }, + }, + }, + }, + { + Name: "GITEA__SERVER__LFS_JWT_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "jwtSecret", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.LfsJwtSecret, + }, + }, + }, + }, { Name: "GITEA__DATABASE__PASSWD", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "password", + Key: "postgresPassword", LocalObjectReference: corev1.LocalObjectReference{ Name: n.Spec.DataStoreParams.DBSecret, }, @@ -233,16 +275,48 @@ func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar { Name: "GITEA_TEMP", Value: "/tmp/gitea", }, - { Name: "HOME", Value: "/data/gitea/git", }, + { + Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "objectStoreKey", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.ObjectStoreSecret, + }, + }, + }, + }, + { + Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "objectStoreSecret", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.ObjectStoreSecret, + }, + }, + }, + }, + { + Name: "GITEA__SERVER__LFS_JWT_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "jwtSecret", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.LfsJwtSecret, + }, + }, + }, + }, { Name: "GITEA__DATABASE__PASSWD", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "password", + Key: "postgresPassword", LocalObjectReference: corev1.LocalObjectReference{ Name: n.Spec.DataStoreParams.DBSecret, }, @@ -253,7 +327,7 @@ func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar { Name: "GITEA_ADMIN_USERNAME", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "username", + Key: "GITEA_ADMIN_USERNAME", LocalObjectReference: corev1.LocalObjectReference{ Name: n.Spec.DataStoreParams.GiteaAdminSecret, }, @@ -264,7 +338,7 @@ func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar { Name: "GITEA_ADMIN_PASSWORD", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "password", + Key: "GITEA_ADMIN_PASSWORD", LocalObjectReference: corev1.LocalObjectReference{ Name: n.Spec.DataStoreParams.GiteaAdminSecret, }, @@ -333,18 +407,67 @@ func (n *NemoDatastoreV2) GetVolumes() []corev1.Volume { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, - { + } + + if n.ShouldEnablePersistentStorage() { + volumes = append(volumes, corev1.Volume{ Name: "data", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: n.Spec.DataStoreParams.PVCSharedData, + ClaimName: n.Spec.DataStoreParams.Persistence.ClaimName, }, }, - }, + }) + } else { + volumes = append(volumes, corev1.Volume{ + Name: "data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) } return volumes } +// GetPVC returns the persistence PVC object for the NeMoDatastoreV2 container +func (n *NemoDatastoreV2) GetPVC() (*corev1.PersistentVolumeClaim, error) { + if n.Spec.DataStoreParams.Persistence == nil { + return nil, nil + } + + persistence := n.Spec.DataStoreParams.Persistence + + size, err := resource.ParseQuantity(persistence.Size) + if err != nil { + return nil, fmt.Errorf("failed to parse size for pvc creation %s, err %v", persistence.ClaimName, err) + } + + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: persistence.ClaimName, + Namespace: n.Namespace, + Labels: persistence.Labels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: persistence.StorageClass, + AccessModes: persistence.AccessModes, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: size, + }, + }, + }, + }, nil +} + +func (n *NemoDatastoreV2) ShouldCreatePersistentStorage() bool { + return n.Spec.DataStoreParams.Persistence != nil && n.Spec.DataStoreParams.Persistence.Create +} + +func (n *NemoDatastoreV2) ShouldEnablePersistentStorage() bool { + return n.Spec.DataStoreParams.Persistence != nil && n.Spec.DataStoreParams.Persistence.Enabled +} + // GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance func (n *NemoDatastoreV2) GetEnvFrom() []corev1.EnvFromSource { return []corev1.EnvFromSource{} @@ -583,14 +706,22 @@ func (n *NemoDatastoreV2) GetInitContainers() []corev1.Container { user := int64(1000) return []corev1.Container{ { - Name: "init-datastore", + Name: "init-directories", Image: n.GetImage(), ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), Command: []string{ - "/bin/sh", "-c", + "/usr/sbin/init/init_directory_structure.sh", }, - Args: []string{ - "/usr/sbin/init/init_directory_structure.sh && /usr/sbin/config_environment.sh", + VolumeMounts: n.GetVolumeMountsInitContainer(), + Env: n.GetInitContainerEnv(), + EnvFrom: n.GetInitAppIniEnvFrom(), + }, + { + Name: "init-app-ini", + Image: n.GetImage(), + ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), + Command: []string{ + "/usr/sbin/config_environment.sh", }, VolumeMounts: n.GetVolumeMountsInitContainer(), Env: n.GetInitContainerEnv(), diff --git a/api/apps/v1alpha1/zz_generated.deepcopy.go b/api/apps/v1alpha1/zz_generated.deepcopy.go index 602a446d..12d4a6a5 100644 --- a/api/apps/v1alpha1/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/zz_generated.deepcopy.go @@ -1234,6 +1234,11 @@ func (in *NemoDatastoreV2List) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NemoDatastoreV2Params) DeepCopyInto(out *NemoDatastoreV2Params) { *out = *in + if in.Persistence != nil { + in, out := &in.Persistence, &out.Persistence + *out = new(Persistence) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Params. @@ -1321,7 +1326,7 @@ func (in *NemoDatastoreV2Spec) DeepCopyInto(out *NemoDatastoreV2Spec) { *out = new(int64) **out = **in } - out.DataStoreParams = in.DataStoreParams + in.DataStoreParams.DeepCopyInto(&out.DataStoreParams) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Spec. @@ -1886,6 +1891,38 @@ func (in *NemoGuardrailStatus) DeepCopy() *NemoGuardrailStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Persistence) DeepCopyInto(out *Persistence) { + *out = *in + if in.AccessModes != nil { + in, out := &in.AccessModes, &out.AccessModes + *out = make([]v1.PersistentVolumeAccessMode, len(*in)) + copy(*out, *in) + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.StorageClass != nil { + in, out := &in.StorageClass, &out.StorageClass + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Persistence. +func (in *Persistence) DeepCopy() *Persistence { + if in == nil { + return nil + } + out := new(Persistence) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PersistentVolumeClaim) DeepCopyInto(out *PersistentVolumeClaim) { *out = *in diff --git a/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml b/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml index d72012e9..518d4f75 100644 --- a/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml +++ b/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml @@ -81,14 +81,34 @@ spec: type: string objStoreSecret: type: string - pvcSharedData: - type: string - size: - type: string + persistence: + properties: + accessModes: + items: + type: string + type: array + claimName: + type: string + create: + type: boolean + enabled: + type: boolean + labels: + additionalProperties: + type: string + type: object + size: + type: string + storageClass: + type: string + required: + - claimName + - create + - enabled + - size + type: object sshEnabled: type: boolean - storageClass: - type: string required: - datastoreConfigSecret - datastoreInitSecret @@ -98,9 +118,7 @@ spec: - giteaAdminSecret - lfsJwtSecret - objStoreSecret - - pvcSharedData - sshEnabled - - storageClass type: object env: items: diff --git a/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml b/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml index d72012e9..518d4f75 100644 --- a/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml +++ b/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml @@ -81,14 +81,34 @@ spec: type: string objStoreSecret: type: string - pvcSharedData: - type: string - size: - type: string + persistence: + properties: + accessModes: + items: + type: string + type: array + claimName: + type: string + create: + type: boolean + enabled: + type: boolean + labels: + additionalProperties: + type: string + type: object + size: + type: string + storageClass: + type: string + required: + - claimName + - create + - enabled + - size + type: object sshEnabled: type: boolean - storageClass: - type: string required: - datastoreConfigSecret - datastoreInitSecret @@ -98,9 +118,7 @@ spec: - giteaAdminSecret - lfsJwtSecret - objStoreSecret - - pvcSharedData - sshEnabled - - storageClass type: object env: items: diff --git a/config/samples/apps_v1alpha1_nimdatastorev2.yaml b/config/samples/apps_v1alpha1_nimdatastorev2.yaml new file mode 100644 index 00000000..6a6555f5 --- /dev/null +++ b/config/samples/apps_v1alpha1_nimdatastorev2.yaml @@ -0,0 +1,65 @@ +apiVersion: apps.nvidia.com/v1alpha1 +kind: NemoDatastoreV2 +metadata: + name: nemodatastore +spec: + authSecret: ngc-image-pull-secret + dataStoreParams: + datastoreConfigSecret: "nemo-ms-nemo-datastore" + datastoreInitSecret: "nemo-ms-nemo-datastore-init" + datastoreInlineConfigSecret: "nemo-ms-nemo-datastore-inline-config" + datastoreSettingsSecret: "nemo-ms-nemo-datastore-setting" + dbSecret: "nds-pg-existing-secret" + giteaAdminSecret: "gitea-admin-credentials" + lfsJwtSecret: "nemo-ms-nemo-datastore--lfs-jwt" + objStoreSecret: "nds-minio-existing-secret" + sshEnabled: false + persistence: + claimName: "pvc-shared-data" + create: true + enabled: true + storageClass: "local-path" + accessModes: + - ReadWriteOnce + size: "10Gi" + expose: + service: + port: 3000 + type: ClusterIP + image: + repository: "nvcr.io/nvidian/nemo-llm/datastore" + tag: "25.01-rc8" + pullPolicy: IfNotPresent + replicas: 1 + resources: + requests: + memory: "256Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "1" + livenessProbe: + enabled: true + probe: + httpGet: + path: /v1/health + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + readinessProbe: + enabled: true + probe: + httpGet: + path: /v1/health + port: 3000 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 3 + # metrics: + # enabled: true + # serviceMonitor: + # additionalLabels: + # app: sample-nemodatastore + # interval: "30s" + # scrapeTimeout: "10s" diff --git a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml index d72012e9..518d4f75 100644 --- a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml +++ b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml @@ -81,14 +81,34 @@ spec: type: string objStoreSecret: type: string - pvcSharedData: - type: string - size: - type: string + persistence: + properties: + accessModes: + items: + type: string + type: array + claimName: + type: string + create: + type: boolean + enabled: + type: boolean + labels: + additionalProperties: + type: string + type: object + size: + type: string + storageClass: + type: string + required: + - claimName + - create + - enabled + - size + type: object sshEnabled: type: boolean - storageClass: - type: string required: - datastoreConfigSecret - datastoreInitSecret @@ -98,9 +118,7 @@ spec: - giteaAdminSecret - lfsJwtSecret - objStoreSecret - - pvcSharedData - sshEnabled - - storageClass type: object env: items: diff --git a/internal/controller/nemo_datastorev2_controller.go b/internal/controller/nemo_datastorev2_controller.go index 3f2ead39..f0020902 100644 --- a/internal/controller/nemo_datastorev2_controller.go +++ b/internal/controller/nemo_datastorev2_controller.go @@ -21,8 +21,6 @@ import ( "fmt" "reflect" - "k8s.io/apimachinery/pkg/api/resource" - appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" "github.com/NVIDIA/k8s-nim-operator/internal/conditions" "github.com/NVIDIA/k8s-nim-operator/internal/k8sutil" @@ -95,7 +93,7 @@ func NewNemoDatastoreV2Reconciler(client client.Client, scheme *runtime.Scheme, // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalars,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -277,24 +275,15 @@ func (r *NemoDatastoreV2Reconciler) reconcileNemoDatastoreV2(ctx context.Context } }() - pvc := corev1.PersistentVolumeClaim{} - pvc.Name = nemoDatastore.Spec.DataStoreParams.PVCSharedData - pvc.Spec.StorageClassName = &nemoDatastore.Spec.DataStoreParams.StorageClass - size, err := resource.ParseQuantity(nemoDatastore.Spec.DataStoreParams.Size) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to parse size for pvc creation %s, err %v", nemoDatastore.Name, err) - } - pvc.Spec.Resources = corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: size, - }, - } - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pvc.Namespace = nemoDatastore.Namespace - - err = r.Create(ctx, &pvc) - if err != nil && !errors.IsAlreadyExists(err) { - logger.Error(err, "Failed to create pvc", "name", pvc.Name) + if nemoDatastore.ShouldCreatePersistentStorage() { + pvc, err := nemoDatastore.GetPVC() + if err != nil { + return ctrl.Result{}, err + } + err = r.Create(ctx, pvc) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "Failed to create pvc", "name", pvc.Name) + } } // Generate annotation for the current operator-version and apply to all resources From 32ff019d98f16ab2b797e049d7b30927b279f5d0 Mon Sep 17 00:00:00 2001 From: Sheng Lin Date: Sat, 18 Jan 2025 18:41:32 -0500 Subject: [PATCH 3/4] Deprecate DataStore v1 Signed-off-by: Sheng Lin --- api/apps/v1alpha1/nemo_datastore_types.go | 415 ++- api/apps/v1alpha1/nemo_datastorev2_types.go | 1104 -------- api/apps/v1alpha1/zz_generated.deepcopy.go | 251 +- .../typed/apps/v1alpha1/apps_client.go | 5 - .../apps/v1alpha1/fake/fake_apps_client.go | 4 - .../v1alpha1/fake/fake_nemodatastorev2.go | 146 -- .../apps/v1alpha1/generated_expansion.go | 2 - .../typed/apps/v1alpha1/nemodatastorev2.go | 68 - .../apps.nvidia.com_nemodatastores.yaml | 66 +- .../apps.nvidia.com_nemodatastorev2s.yaml | 2230 ----------------- cmd/main.go | 17 +- .../bases/apps.nvidia.com_nemodatastores.yaml | 66 +- .../apps.nvidia.com_nemodatastorev2s.yaml | 2230 ----------------- config/rbac/role.yaml | 3 - ...2.yaml => apps_v1alpha1_nimdatastore.yaml} | 10 +- .../crds/apps.nvidia.com_nemodatastores.yaml | 66 +- .../apps.nvidia.com_nemodatastorev2s.yaml | 2230 ----------------- .../controller/nemo_datastore_controller.go | 46 + .../controller/nemo_datastorev2_controller.go | 631 ----- 19 files changed, 544 insertions(+), 9046 deletions(-) delete mode 100644 api/apps/v1alpha1/nemo_datastorev2_types.go delete mode 100644 api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go delete mode 100644 api/versioned/typed/apps/v1alpha1/nemodatastorev2.go delete mode 100644 bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml delete mode 100644 config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml rename config/samples/{apps_v1alpha1_nimdatastorev2.yaml => apps_v1alpha1_nimdatastore.yaml} (91%) delete mode 100644 deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml delete mode 100644 internal/controller/nemo_datastorev2_controller.go diff --git a/api/apps/v1alpha1/nemo_datastore_types.go b/api/apps/v1alpha1/nemo_datastore_types.go index 884cb677..f80be4c0 100644 --- a/api/apps/v1alpha1/nemo_datastore_types.go +++ b/api/apps/v1alpha1/nemo_datastore_types.go @@ -30,6 +30,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! @@ -78,7 +79,7 @@ type NemoDatastoreSpec struct { GroupID *int64 `json:"groupID,omitempty"` RuntimeClass string `json:"runtimeClass,omitempty"` - DataStoreParams NemoDataStoreParams `json:"dataStoreParams"` + DataStoreParams NemoDatastoreParams `json:"dataStoreParams"` } // NemoDatastoreStatus defines the observed state of NemoDatastore @@ -88,20 +89,21 @@ type NemoDatastoreStatus struct { State string `json:"state,omitempty"` } -type NemoDataStoreParams struct { - AppVersion string `json:"appVersion"` - GiteaEndpoint string `json:"giteaEndpoint"` - GiteaSecret string `json:"giteaSecret"` - DatabaseURL string `json:"databaseURL"` - DatabaseHost string `json:"databaseHost"` - DatabasePort string `json:"databasePort"` - DBSecret string `json:"dbSecret"` +type NemoDatastoreParams struct { + DBSecret string `json:"dbSecret"` + GiteaAdminSecret string `json:"giteaAdminSecret"` - EnvConfigMap string `json:"envConfigmap"` - EnvSecret string `json:"envSecret"` + ObjectStoreSecret string `json:"objStoreSecret"` + DataStoreSettingsSecret string `json:"datastoreSettingsSecret"` + LfsJwtSecret string `json:"lfsJwtSecret"` - InitContainerImage string `json:"initContainerImage,omitempty"` - InitContainerCommand []string `json:"initContainerCommand,omitempty"` + DataStoreInitSecret string `json:"datastoreInitSecret"` + DataStoreConfigSecret string `json:"datastoreConfigSecret"` + DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"` + + SshEnabled bool `json:"sshEnabled"` + + PVC *PersistentVolumeClaim `json:"pvc,omitempty"` } // +genclient @@ -130,10 +132,11 @@ type NemoDatastoreList struct { // GetPVCName returns the name to be used for the PVC based on the custom spec // Prefers pvc.Name if explicitly set by the user in the NemoDatastore instance -func (n *NemoDatastore) GetPVCName(pvc PersistentVolumeClaim) string { +func (n *NemoDatastore) GetPVCName() string { pvcName := fmt.Sprintf("%s-pvc", n.GetName()) - if pvc.Name != "" { - pvcName = pvc.Name + dsParam := n.Spec.DataStoreParams + if dsParam.PVC != nil && dsParam.PVC.Name != "" { + pvcName = dsParam.PVC.Name } return pvcName } @@ -157,97 +160,284 @@ func (n *NemoDatastore) GetStandardLabels() map[string]string { } } -// GetStandardEnv returns the standard set of env variables for the NemoDatastore container +// GetMainContainerEnv returns the standard set of env variables for the NemoDatastore main container func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { // add standard env required for NIM service envVars := []corev1.EnvVar{ { - Name: "APP_VERSION", - Value: n.Spec.DataStoreParams.AppVersion, + Name: "SSH_LISTEN_PORT", + Value: "2222", + }, + { + Name: "SSH_PORT", + Value: "22", + }, + { + Name: "GITEA_APP_INI", + Value: "/data/gitea/conf/app.ini", }, { - Name: "GITEA_ENDPOINT", - Value: n.Spec.DataStoreParams.GiteaEndpoint, + Name: "GITEA_CUSTOM", + Value: "/data/gitea", }, { - Name: "GITEA_ORG_NAME", + Name: "GITEA_WORK_DIR", + Value: "/data", + }, + { + Name: "TMPDIR", + Value: "/tmp/gitea", + }, + { + Name: "GITEA_TEMP", + Value: "/tmp/gitea", + }, + { + Name: "HOME", + Value: "/data/gitea/git", + }, + { + Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "username", + Key: "objectStoreKey", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.GiteaSecret, + Name: n.Spec.DataStoreParams.ObjectStoreSecret, }, }, }, }, { - Name: "GITEA_PASSWORD", + Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "password", + Key: "objectStoreSecret", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.GiteaSecret, + Name: n.Spec.DataStoreParams.ObjectStoreSecret, }, }, }, }, { - Name: "DB_PASSWORD", + Name: "GITEA__SERVER__LFS_JWT_SECRET", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "password", + Key: "jwtSecret", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DBSecret, + Name: n.Spec.DataStoreParams.LfsJwtSecret, }, }, }, }, { - Name: "DATABASE_URL", - Value: n.Spec.DataStoreParams.DatabaseURL, + Name: "GITEA__DATABASE__PASSWD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "postgresPassword", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DBSecret, + }, + }, + }, }, } return envVars } -// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance -func (n *NemoDatastore) GetEnvFrom() []corev1.EnvFromSource { - return []corev1.EnvFromSource{ +func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { + envVars := []corev1.EnvVar{ { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.EnvConfigMap, + Name: "GITEA_APP_INI", + Value: "/data/gitea/conf/app.ini", + }, + { + Name: "GITEA_CUSTOM", + Value: "/data/gitea", + }, + { + Name: "GITEA_WORK_DIR", + Value: "/data", + }, + { + Name: "TMPDIR", + Value: "/tmp/gitea", + }, + { + Name: "GITEA_TEMP", + Value: "/tmp/gitea", + }, + { + Name: "HOME", + Value: "/data/gitea/git", + }, + { + Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "objectStoreKey", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.ObjectStoreSecret, + }, }, }, }, { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.EnvSecret, + Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "objectStoreSecret", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.ObjectStoreSecret, + }, + }, + }, + }, + { + Name: "GITEA__SERVER__LFS_JWT_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "jwtSecret", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.LfsJwtSecret, + }, + }, + }, + }, + { + Name: "GITEA__DATABASE__PASSWD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "postgresPassword", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DBSecret, + }, + }, + }, + }, + { + Name: "GITEA_ADMIN_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "GITEA_ADMIN_USERNAME", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.GiteaAdminSecret, + }, + }, + }, + }, + { + Name: "GITEA_ADMIN_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "GITEA_ADMIN_PASSWORD", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.GiteaAdminSecret, + }, }, }, }, } + return envVars } -func (n *NemoDatastore) GetInitContainers() []corev1.Container { - image := n.Spec.DataStoreParams.InitContainerImage - if image == "" { - image = "busybox" - } - cmd := n.Spec.DataStoreParams.InitContainerCommand - if len(cmd) == 0 { - cmd = []string{ - "sh", - "-c", - fmt.Sprintf("until nc -z %s %s; do echo \"PostgreSQL is unavailable. Sleeping for 5 seconds\"; sleep 5; done;", n.Spec.DataStoreParams.DatabaseHost, n.Spec.DataStoreParams.DatabasePort), - } +// GetVolumes returns volumes for the NemoDatastore container +func (n *NemoDatastore) GetVolumes() []corev1.Volume { + /*volumes: + - name: init + secret: + defaultMode: 110 + secretName: datastore-nemo-datastore-init + - name: config + secret: + defaultMode: 110 + secretName: datastore-nemo-datastore + - name: inline-config-sources + secret: + defaultMode: 420 + secretName: datastore-nemo-datastore-inline-config + - emptyDir: {} + name: temp + - name: data + persistentVolumeClaim: + claimName: datastore-shared-storage + */ + var initMode = int32(110) + var configMode = int32(420) + + volumes := []corev1.Volume{ + { + Name: "init", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreInitSecret, + DefaultMode: &initMode, + }, + }, + }, + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreConfigSecret, + DefaultMode: &initMode, + }, + }, + }, + { + Name: "inline-config-sources", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreInlineConfigSecret, + DefaultMode: &configMode, + }, + }, + }, + { + Name: "temp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, } - return []corev1.Container{ + + if n.Spec.DataStoreParams.PVC != nil { + volumes = append(volumes, corev1.Volume{ + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: n.GetPVCName(), + }, + }, + }) + } else { + volumes = append(volumes, corev1.Volume{ + Name: "data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + } + return volumes +} + +func (n *NemoDatastore) ShouldCreatePersistentStorage() bool { + return n.Spec.DataStoreParams.PVC != nil && n.Spec.DataStoreParams.PVC.Create != nil && *n.Spec.DataStoreParams.PVC.Create +} + +// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance +func (n *NemoDatastore) GetEnvFrom() []corev1.EnvFromSource { + return []corev1.EnvFromSource{} +} + +// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance +func (n *NemoDatastore) GetInitAppIniEnvFrom() []corev1.EnvFromSource { + return []corev1.EnvFromSource{ { - Name: "wait-postgres-ready", - Image: image, - Command: cmd, + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DataStoreSettingsSecret, + }, + }, }, } } @@ -429,15 +619,100 @@ func (n *NemoDatastore) GetDefaultStartupProbe() *corev1.Probe { return &probe } -// GetVolumes returns volumes for the NemoDatastore container -func (n *NemoDatastore) GetVolumes() []corev1.Volume { - volumes := []corev1.Volume{} - return volumes -} - // GetVolumeMounts returns volumes for the NemoDatastore container func (n *NemoDatastore) GetVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{} + mounts := []corev1.VolumeMount{ + { + MountPath: "/tmp", + Name: "temp", + }, + } + + dataMount := corev1.VolumeMount{ + MountPath: "/data", + Name: "data", + } + + if n.Spec.DataStoreParams.PVC != nil { + dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath + } + mounts = append(mounts, dataMount) + return mounts +} + +func (n *NemoDatastore) GetVolumeMountsInitContainer() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + { + MountPath: "/usr/sbin", + Name: "config", + }, + { + MountPath: "/tmp", + Name: "temp", + }, + { + MountPath: "/env-to-ini-mounts/inlines/", + Name: "inline-config-sources", + }, + { + MountPath: "/usr/sbin/init", + Name: "init", + }, + } + dataMount := corev1.VolumeMount{ + MountPath: "/data", + Name: "data", + } + + if n.Spec.DataStoreParams.PVC != nil { + dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath + } + mounts = append(mounts, dataMount) + return mounts +} + +func (n *NemoDatastore) GetInitContainers() []corev1.Container { + return []corev1.Container{ + { + Name: "init-directories", + Image: n.GetImage(), + ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), + Command: []string{ + "/usr/sbin/init/init_directory_structure.sh", + }, + VolumeMounts: n.GetVolumeMountsInitContainer(), + Env: n.GetInitContainerEnv(), + EnvFrom: n.GetInitAppIniEnvFrom(), + }, + { + Name: "init-app-ini", + Image: n.GetImage(), + ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), + Command: []string{ + "/usr/sbin/config_environment.sh", + }, + VolumeMounts: n.GetVolumeMountsInitContainer(), + Env: n.GetInitContainerEnv(), + EnvFrom: n.GetInitAppIniEnvFrom(), + }, + { + Name: "configure-datastore", + Image: n.GetImage(), + ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), + Command: []string{ + "/bin/sh", "-c", + }, + Args: []string{ + "/usr/sbin/init/configure_gitea.sh", + }, + VolumeMounts: n.GetVolumeMountsInitContainer(), + Env: n.GetInitContainerEnv(), + EnvFrom: n.GetInitAppIniEnvFrom(), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: n.GetUserID(), + }, + }, + } } // GetServiceAccountName returns service account name for the NemoDatastore deployment @@ -505,14 +780,18 @@ func (n *NemoDatastore) GetServiceType() string { // GetUserID returns the user ID for the NemoDatastore deployment func (n *NemoDatastore) GetUserID() *int64 { - return n.Spec.UserID - + if n.Spec.UserID != nil { + return n.Spec.UserID + } + return ptr.To[int64](1000) } // GetGroupID returns the group ID for the NemoDatastore deployment func (n *NemoDatastore) GetGroupID() *int64 { - return n.Spec.GroupID - + if n.Spec.GroupID != nil { + return n.Spec.GroupID + } + return ptr.To[int64](2000) } // GetServiceAccountParams return params to render ServiceAccount from templates diff --git a/api/apps/v1alpha1/nemo_datastorev2_types.go b/api/apps/v1alpha1/nemo_datastorev2_types.go deleted file mode 100644 index cddac0a2..00000000 --- a/api/apps/v1alpha1/nemo_datastorev2_types.go +++ /dev/null @@ -1,1104 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "fmt" - "maps" - "os" - - rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types" - utils "github.com/NVIDIA/k8s-nim-operator/internal/utils" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - -const ( - // NemoDatastoreV2ConditionReady indicates that the NEMO datastore service is ready. - NemoDatastoreV2ConditionReady = "Ready" - // NemoDatastoreV2ConditionFailed indicates that the NEMO datastore service has failed. - NemoDatastoreV2ConditionFailed = "Failed" - - // NemoDatastoreV2StatusPending indicates that NEMO datastore service is in pending state - NemoDatastoreV2StatusPending = "Pending" - // NemoDatastoreV2StatusNotReady indicates that NEMO datastore service is not ready - NemoDatastoreV2StatusNotReady = "NotReady" - // NemoDatastoreV2StatusReady indicates that NEMO datastore service is ready - NemoDatastoreV2StatusReady = "Ready" - // NemoDatastoreV2StatusFailed indicates that NEMO datastore service has failed - NemoDatastoreV2StatusFailed = "Failed" -) - -// NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 -type NemoDatastoreV2Spec struct { - Image Image `json:"image,omitempty"` - Command []string `json:"command,omitempty"` - Args []string `json:"args,omitempty"` - Env []corev1.EnvVar `json:"env,omitempty"` - // The name of an secret that contains authn for the NGC NIM service API - AuthSecret string `json:"authSecret"` - Labels map[string]string `json:"labels,omitempty"` - Annotations map[string]string `json:"annotations,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - PodAffinity *corev1.PodAffinity `json:"podAffinity,omitempty"` - Resources *corev1.ResourceRequirements `json:"resources,omitempty"` - Expose Expose `json:"expose,omitempty"` - LivenessProbe Probe `json:"livenessProbe,omitempty"` - ReadinessProbe Probe `json:"readinessProbe,omitempty"` - StartupProbe Probe `json:"startupProbe,omitempty"` - Scale Autoscaling `json:"scale,omitempty"` - Metrics Metrics `json:"metrics,omitempty"` - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:default:=1 - Replicas int `json:"replicas,omitempty"` - UserID *int64 `json:"userID,omitempty"` - GroupID *int64 `json:"groupID,omitempty"` - RuntimeClass string `json:"runtimeClass,omitempty"` - - DataStoreParams NemoDatastoreV2Params `json:"dataStoreParams"` -} - -// NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 -type NemoDatastoreV2Status struct { - Conditions []metav1.Condition `json:"conditions,omitempty"` - AvailableReplicas int32 `json:"availableReplicas,omitempty"` - State string `json:"state,omitempty"` -} - -type NemoDatastoreV2Params struct { - DBSecret string `json:"dbSecret"` - GiteaAdminSecret string `json:"giteaAdminSecret"` - - ObjectStoreSecret string `json:"objStoreSecret"` - DataStoreSettingsSecret string `json:"datastoreSettingsSecret"` - LfsJwtSecret string `json:"lfsJwtSecret"` - - DataStoreInitSecret string `json:"datastoreInitSecret"` - DataStoreConfigSecret string `json:"datastoreConfigSecret"` - DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"` - - SshEnabled bool `json:"sshEnabled"` - - Persistence *Persistence `json:"persistence,omitempty"` -} - -type Persistence struct { - Create bool `json:"create"` - Enabled bool `json:"enabled"` - ClaimName string `json:"claimName"` - Size string `json:"size"` - AccessModes []corev1.PersistentVolumeAccessMode `json:"accessModes,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - StorageClass *string `json:"storageClass,omitempty"` -} - -// +genclient -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0 -// +kubebuilder:printcolumn:name="Age",type="date",format="date-time",JSONPath=".metadata.creationTimestamp",priority=0 - -// NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API -type NemoDatastoreV2 struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec NemoDatastoreV2Spec `json:"spec,omitempty"` - Status NemoDatastoreV2Status `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// NemoDatastoreV2List contains a list of NemoDatastoreV2 -type NemoDatastoreV2List struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []NemoDatastoreV2 `json:"items"` -} - -// GetPVCName returns the name to be used for the PVC based on the custom spec -// Prefers pvc.Name if explicitly set by the user in the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetPVCName(pvc PersistentVolumeClaim) string { - pvcName := fmt.Sprintf("%s-pvc", n.GetName()) - if pvc.Name != "" { - pvcName = pvc.Name - } - return pvcName -} - -// GetStandardSelectorLabels returns the standard selector labels for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetStandardSelectorLabels() map[string]string { - return map[string]string{ - "app.kubernetes.io/name": n.Name, - "app.kubernetes.io/instance": n.Name, - } -} - -// GetStandardLabels returns the standard set of labels for NemoDatastoreV2 resources -func (n *NemoDatastoreV2) GetStandardLabels() map[string]string { - return map[string]string{ - "app.kubernetes.io/name": n.Name, - "app.kubernetes.io/instance": n.Name, - "app.kubernetes.io/operator-version": os.Getenv("OPERATOR_VERSION"), - "app.kubernetes.io/part-of": "nemo-datastore-service", - "app.kubernetes.io/managed-by": "k8s-nim-operator", - } -} - -// GetMainContainerEnv returns the standard set of env variables for the NemoDatastoreV2 main container -func (n *NemoDatastoreV2) GetStandardEnv() []corev1.EnvVar { - // add standard env required for NIM service - envVars := []corev1.EnvVar{ - { - Name: "SSH_LISTEN_PORT", - Value: "2222", - }, - { - Name: "SSH_PORT", - Value: "22", - }, - { - Name: "GITEA_APP_INI", - Value: "/data/gitea/conf/app.ini", - }, - { - Name: "GITEA_CUSTOM", - Value: "/data/gitea", - }, - { - Name: "GITEA_WORK_DIR", - Value: "/data", - }, - { - Name: "TMPDIR", - Value: "/tmp/gitea", - }, - { - Name: "GITEA_TEMP", - Value: "/tmp/gitea", - }, - { - Name: "HOME", - Value: "/data/gitea/git", - }, - { - Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreKey", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, - }, - }, - }, - }, - { - Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreSecret", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, - }, - }, - }, - }, - { - Name: "GITEA__SERVER__LFS_JWT_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "jwtSecret", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.LfsJwtSecret, - }, - }, - }, - }, - { - Name: "GITEA__DATABASE__PASSWD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "postgresPassword", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DBSecret, - }, - }, - }, - }, - } - return envVars -} - -func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar { - envVars := []corev1.EnvVar{ - { - Name: "GITEA_APP_INI", - Value: "/data/gitea/conf/app.ini", - }, - { - Name: "GITEA_CUSTOM", - Value: "/data/gitea", - }, - { - Name: "GITEA_WORK_DIR", - Value: "/data", - }, - { - Name: "TMPDIR", - Value: "/tmp/gitea", - }, - { - Name: "GITEA_TEMP", - Value: "/tmp/gitea", - }, - { - Name: "HOME", - Value: "/data/gitea/git", - }, - { - Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreKey", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, - }, - }, - }, - }, - { - Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreSecret", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, - }, - }, - }, - }, - { - Name: "GITEA__SERVER__LFS_JWT_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "jwtSecret", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.LfsJwtSecret, - }, - }, - }, - }, - { - Name: "GITEA__DATABASE__PASSWD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "postgresPassword", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DBSecret, - }, - }, - }, - }, - { - Name: "GITEA_ADMIN_USERNAME", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "GITEA_ADMIN_USERNAME", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.GiteaAdminSecret, - }, - }, - }, - }, - { - Name: "GITEA_ADMIN_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "GITEA_ADMIN_PASSWORD", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.GiteaAdminSecret, - }, - }, - }, - }, - } - return envVars -} - -// GetVolumes returns volumes for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetVolumes() []corev1.Volume { - /*volumes: - - name: init - secret: - defaultMode: 110 - secretName: datastore-v2-nemo-datastore-init - - name: config - secret: - defaultMode: 110 - secretName: datastore-v2-nemo-datastore - - name: inline-config-sources - secret: - defaultMode: 420 - secretName: datastore-v2-nemo-datastore-inline-config - - emptyDir: {} - name: temp - - name: data - persistentVolumeClaim: - claimName: datastore-shared-storage - */ - var initMode = int32(110) - var configMode = int32(420) - - volumes := []corev1.Volume{ - { - Name: "init", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreInitSecret, - DefaultMode: &initMode, - }, - }, - }, - { - Name: "config", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreConfigSecret, - DefaultMode: &initMode, - }, - }, - }, - { - Name: "inline-config-sources", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreInlineConfigSecret, - DefaultMode: &configMode, - }, - }, - }, - { - Name: "temp", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - } - - if n.ShouldEnablePersistentStorage() { - volumes = append(volumes, corev1.Volume{ - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: n.Spec.DataStoreParams.Persistence.ClaimName, - }, - }, - }) - } else { - volumes = append(volumes, corev1.Volume{ - Name: "data", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }) - } - return volumes -} - -// GetPVC returns the persistence PVC object for the NeMoDatastoreV2 container -func (n *NemoDatastoreV2) GetPVC() (*corev1.PersistentVolumeClaim, error) { - if n.Spec.DataStoreParams.Persistence == nil { - return nil, nil - } - - persistence := n.Spec.DataStoreParams.Persistence - - size, err := resource.ParseQuantity(persistence.Size) - if err != nil { - return nil, fmt.Errorf("failed to parse size for pvc creation %s, err %v", persistence.ClaimName, err) - } - - return &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: persistence.ClaimName, - Namespace: n.Namespace, - Labels: persistence.Labels, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - StorageClassName: persistence.StorageClass, - AccessModes: persistence.AccessModes, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: size, - }, - }, - }, - }, nil -} - -func (n *NemoDatastoreV2) ShouldCreatePersistentStorage() bool { - return n.Spec.DataStoreParams.Persistence != nil && n.Spec.DataStoreParams.Persistence.Create -} - -func (n *NemoDatastoreV2) ShouldEnablePersistentStorage() bool { - return n.Spec.DataStoreParams.Persistence != nil && n.Spec.DataStoreParams.Persistence.Enabled -} - -// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetEnvFrom() []corev1.EnvFromSource { - return []corev1.EnvFromSource{} -} - -// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetInitAppIniEnvFrom() []corev1.EnvFromSource { - return []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DataStoreSettingsSecret, - }, - }, - }, - } -} - -// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetStandardAnnotations() map[string]string { - standardAnnotations := map[string]string{ - "openshift.io/scc": "nonroot", - } - return standardAnnotations -} - -// GetNemoDatastoreV2Annotations returns annotations to apply to the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetNemoDatastoreV2Annotations() map[string]string { - standardAnnotations := n.GetStandardAnnotations() - - if n.Spec.Annotations != nil { - return utils.MergeMaps(standardAnnotations, n.Spec.Annotations) - } - - return standardAnnotations -} - -// GetServiceLabels returns merged labels to apply to the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetServiceLabels() map[string]string { - standardLabels := n.GetStandardLabels() - - if n.Spec.Labels != nil { - return utils.MergeMaps(standardLabels, n.Spec.Labels) - } - return standardLabels -} - -// GetSelectorLabels returns standard selector labels to apply to the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetSelectorLabels() map[string]string { - // TODO: add custom ones - return n.GetStandardSelectorLabels() -} - -// GetNodeSelector returns node selector labels for the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetNodeSelector() map[string]string { - return n.Spec.NodeSelector -} - -// GetTolerations returns tolerations for the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetTolerations() []corev1.Toleration { - return n.Spec.Tolerations -} - -// GetPodAffinity returns pod affinity for the NemoDatastoreV2 instance -func (n *NemoDatastoreV2) GetPodAffinity() *corev1.PodAffinity { - return n.Spec.PodAffinity -} - -// GetContainerName returns name of the container for NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetContainerName() string { - return fmt.Sprintf("%s-ctr", n.Name) -} - -// GetCommand return command to override for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetCommand() []string { - return n.Spec.Command -} - -// GetArgs return arguments for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetArgs() []string { - return n.Spec.Args -} - -// GetEnv returns merged slice of standard and user specified env variables -func (n *NemoDatastoreV2) GetEnv() []corev1.EnvVar { - return utils.MergeEnvVars(n.GetStandardEnv(), n.Spec.Env) -} - -// GetImage returns container image for the NemoDatastoreV2 -func (n *NemoDatastoreV2) GetImage() string { - return fmt.Sprintf("%s:%s", n.Spec.Image.Repository, n.Spec.Image.Tag) -} - -// GetImagePullSecrets returns the image pull secrets for the NIM container -func (n *NemoDatastoreV2) GetImagePullSecrets() []string { - return n.Spec.Image.PullSecrets -} - -// GetImagePullPolicy returns the image pull policy for the NIM container -func (n *NemoDatastoreV2) GetImagePullPolicy() string { - return n.Spec.Image.PullPolicy -} - -// GetResources returns resources to allocate to the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetResources() *corev1.ResourceRequirements { - return n.Spec.Resources -} - -// GetLivenessProbe returns liveness probe for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetLivenessProbe() *corev1.Probe { - if n.Spec.LivenessProbe.Probe == nil { - return n.GetDefaultLivenessProbe() - } - return n.Spec.LivenessProbe.Probe -} - -// GetDefaultLivenessProbe returns the default liveness probe for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetDefaultLivenessProbe() *corev1.Probe { - probe := corev1.Probe{ - InitialDelaySeconds: 15, - TimeoutSeconds: 1, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/v1/health/live", - Port: intstr.IntOrString{ - Type: intstr.Type(0), - IntVal: n.Spec.Expose.Service.Port, - }, - }, - }, - } - - return &probe -} - -// GetReadinessProbe returns readiness probe for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetReadinessProbe() *corev1.Probe { - if n.Spec.ReadinessProbe.Probe == nil { - return n.GetDefaultReadinessProbe() - } - return n.Spec.ReadinessProbe.Probe -} - -// GetDefaultReadinessProbe returns the default readiness probe for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetDefaultReadinessProbe() *corev1.Probe { - probe := corev1.Probe{ - InitialDelaySeconds: 15, - TimeoutSeconds: 1, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/v1/health/ready", - Port: intstr.IntOrString{ - Type: intstr.Type(0), - IntVal: n.Spec.Expose.Service.Port, - }, - }, - }, - } - - return &probe -} - -// GetStartupProbe returns startup probe for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetStartupProbe() *corev1.Probe { - return n.Spec.StartupProbe.Probe -} - -// GetDefaultStartupProbe returns the default startup probe for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetDefaultStartupProbe() *corev1.Probe { - probe := corev1.Probe{ - InitialDelaySeconds: 40, - TimeoutSeconds: 1, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 180, - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/v1/health/ready", - Port: intstr.IntOrString{ - Type: intstr.Type(0), - IntVal: n.Spec.Expose.Service.Port, - }, - }, - }, - } - - return &probe -} - -// GetVolumeMounts returns volumes for the NemoDatastoreV2 container -func (n *NemoDatastoreV2) GetVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{ - { - MountPath: "/tmp", - Name: "temp", - }, - { - MountPath: "/data", - Name: "data", - }, - } -} - -func (n *NemoDatastoreV2) GetVolumeMountsInitContainer() []corev1.VolumeMount { - return []corev1.VolumeMount{ - { - MountPath: "/usr/sbin", - Name: "config", - }, - { - MountPath: "/tmp", - Name: "temp", - }, - { - MountPath: "/data", - Name: "data", - }, - { - MountPath: "/env-to-ini-mounts/inlines/", - Name: "inline-config-sources", - }, - { - MountPath: "/usr/sbin/init", - Name: "init", - }, - } -} - -func (n *NemoDatastoreV2) GetInitContainers() []corev1.Container { - user := int64(1000) - return []corev1.Container{ - { - Name: "init-directories", - Image: n.GetImage(), - ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), - Command: []string{ - "/usr/sbin/init/init_directory_structure.sh", - }, - VolumeMounts: n.GetVolumeMountsInitContainer(), - Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), - }, - { - Name: "init-app-ini", - Image: n.GetImage(), - ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), - Command: []string{ - "/usr/sbin/config_environment.sh", - }, - VolumeMounts: n.GetVolumeMountsInitContainer(), - Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), - }, - { - Name: "configure-datastore", - Image: n.GetImage(), - ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), - Command: []string{ - "/bin/sh", "-c", - }, - Args: []string{ - "/usr/sbin/init/configure_gitea.sh", - }, - VolumeMounts: n.GetVolumeMountsInitContainer(), - Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &user, - }, - }, - } -} - -// GetServiceAccountName returns service account name for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetServiceAccountName() string { - return n.Name -} - -// GetRuntimeClass return the runtime class name for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetRuntimeClass() string { - return n.Spec.RuntimeClass -} - -// GetHPA returns the HPA spec for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetHPA() HorizontalPodAutoscalerSpec { - return n.Spec.Scale.HPA -} - -// GetServiceMonitor returns the Service Monitor details for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetServiceMonitor() ServiceMonitor { - return n.Spec.Metrics.ServiceMonitor -} - -// GetReplicas returns replicas for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetReplicas() int { - if n.IsAutoScalingEnabled() { - return 0 - } - return n.Spec.Replicas -} - -// GetDeploymentKind returns the kind of deployment for NemoDatastoreV2 -func (n *NemoDatastoreV2) GetDeploymentKind() string { - return "Deployment" -} - -// IsAutoScalingEnabled returns true if autoscaling is enabled for NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) IsAutoScalingEnabled() bool { - return n.Spec.Scale.Enabled != nil && *n.Spec.Scale.Enabled -} - -// IsIngressEnabled returns true if ingress is enabled for NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) IsIngressEnabled() bool { - return n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled -} - -// GetIngressSpec returns the Ingress spec NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetIngressSpec() networkingv1.IngressSpec { - return n.Spec.Expose.Ingress.Spec -} - -// IsServiceMonitorEnabled returns true if servicemonitor is enabled for NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) IsServiceMonitorEnabled() bool { - return n.Spec.Metrics.Enabled != nil && *n.Spec.Metrics.Enabled -} - -// GetServicePort returns the service port for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetServicePort() int32 { - return n.Spec.Expose.Service.Port -} - -// GetServiceType returns the service type for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetServiceType() string { - return string(n.Spec.Expose.Service.Type) -} - -// GetUserID returns the user ID for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetUserID() *int64 { - return n.Spec.UserID - -} - -// GetGroupID returns the group ID for the NemoDatastoreV2 deployment -func (n *NemoDatastoreV2) GetGroupID() *int64 { - return n.Spec.GroupID - -} - -// GetServiceAccountParams return params to render ServiceAccount from templates -func (n *NemoDatastoreV2) GetServiceAccountParams() *rendertypes.ServiceAccountParams { - params := &rendertypes.ServiceAccountParams{} - - // Set metadata - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - params.Labels = n.GetServiceLabels() - params.Annotations = n.GetNemoDatastoreV2Annotations() - return params -} - -// GetDeploymentParams returns params to render Deployment from templates -func (n *NemoDatastoreV2) GetDeploymentParams() *rendertypes.DeploymentParams { - params := &rendertypes.DeploymentParams{} - - // Set metadata - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - params.Labels = n.GetServiceLabels() - params.Annotations = n.GetNemoDatastoreV2Annotations() - - // Set template spec - if !n.IsAutoScalingEnabled() { - params.Replicas = n.GetReplicas() - } - params.NodeSelector = n.GetNodeSelector() - params.Tolerations = n.GetTolerations() - params.Affinity = n.GetPodAffinity() - params.ImagePullSecrets = n.GetImagePullSecrets() - params.ImagePullPolicy = n.GetImagePullPolicy() - - // Set labels and selectors - params.SelectorLabels = n.GetSelectorLabels() - - // Set container spec - params.ContainerName = n.GetContainerName() - params.Env = n.GetEnv() - params.Args = n.GetArgs() - params.Command = n.GetCommand() - params.Resources = n.GetResources() - params.Image = n.GetImage() - - // Set container probes - if IsProbeEnabled(n.Spec.LivenessProbe) { - params.LivenessProbe = n.GetLivenessProbe() - } - if IsProbeEnabled(n.Spec.ReadinessProbe) { - params.ReadinessProbe = n.GetReadinessProbe() - } - if IsProbeEnabled(n.Spec.StartupProbe) { - params.StartupProbe = n.GetStartupProbe() - } - params.UserID = n.GetUserID() - params.GroupID = n.GetGroupID() - - // Set service account - params.ServiceAccountName = n.GetServiceAccountName() - - // Set runtime class - params.RuntimeClassName = n.GetRuntimeClass() - return params -} - -// GetStatefulSetParams returns params to render StatefulSet from templates -func (n *NemoDatastoreV2) GetStatefulSetParams() *rendertypes.StatefulSetParams { - - params := &rendertypes.StatefulSetParams{} - - // Set metadata - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - params.Labels = n.GetServiceLabels() - params.Annotations = n.GetNemoDatastoreV2Annotations() - - // Set template spec - if !n.IsAutoScalingEnabled() { - params.Replicas = n.GetReplicas() - } - params.ServiceName = n.GetName() - params.NodeSelector = n.GetNodeSelector() - params.Tolerations = n.GetTolerations() - params.Affinity = n.GetPodAffinity() - params.ImagePullSecrets = n.GetImagePullSecrets() - params.ImagePullPolicy = n.GetImagePullPolicy() - - // Set labels and selectors - params.SelectorLabels = n.GetSelectorLabels() - - // Set container spec - params.ContainerName = n.GetContainerName() - params.Env = n.GetEnv() - params.Args = n.GetArgs() - params.Command = n.GetCommand() - params.Resources = n.GetResources() - params.Image = n.GetImage() - - // Set container probes - params.LivenessProbe = n.GetLivenessProbe() - params.ReadinessProbe = n.GetReadinessProbe() - params.StartupProbe = n.GetStartupProbe() - - // Set service account - params.ServiceAccountName = n.GetServiceAccountName() - - // Set runtime class - params.RuntimeClassName = n.GetRuntimeClass() - return params -} - -// GetServiceParams returns params to render Service from templates -func (n *NemoDatastoreV2) GetServiceParams() *rendertypes.ServiceParams { - params := &rendertypes.ServiceParams{} - - // Set metadata - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - params.Labels = n.GetServiceLabels() - params.Annotations = n.GetServiceAnnotations() - - // Set service selector labels - params.SelectorLabels = n.GetSelectorLabels() - - // Set service type - params.Type = n.GetServiceType() - - // Set service ports - params.Port = n.GetServicePort() - params.PortName = "service-port" - return params -} - -// GetIngressParams returns params to render Ingress from templates -func (n *NemoDatastoreV2) GetIngressParams() *rendertypes.IngressParams { - params := &rendertypes.IngressParams{} - - params.Enabled = n.IsIngressEnabled() - // Set metadata - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - params.Labels = n.GetServiceLabels() - params.Annotations = n.GetIngressAnnotations() - params.Spec = n.GetIngressSpec() - return params -} - -// GetRoleParams returns params to render Role from templates -func (n *NemoDatastoreV2) GetRoleParams() *rendertypes.RoleParams { - params := &rendertypes.RoleParams{} - - // Set metadata - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - - // Set rules to use SCC - params.Rules = []rbacv1.PolicyRule{ - { - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - ResourceNames: []string{"nonroot"}, - Verbs: []string{"use"}, - }, - } - - return params -} - -// GetRoleBindingParams returns params to render RoleBinding from templates -func (n *NemoDatastoreV2) GetRoleBindingParams() *rendertypes.RoleBindingParams { - params := &rendertypes.RoleBindingParams{} - - // Set metadata - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - - params.ServiceAccountName = n.GetServiceAccountName() - params.RoleName = n.GetName() - return params -} - -// GetHPAParams returns params to render HPA from templates -func (n *NemoDatastoreV2) GetHPAParams() *rendertypes.HPAParams { - params := &rendertypes.HPAParams{} - - params.Enabled = n.IsAutoScalingEnabled() - - // Set metadata - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - params.Labels = n.GetServiceLabels() - params.Annotations = n.GetHPAAnnotations() - - // Set HPA spec - hpa := n.GetHPA() - hpaSpec := autoscalingv2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ - Kind: n.GetDeploymentKind(), - Name: n.GetName(), - APIVersion: "apps/v1", - }, - MinReplicas: hpa.MinReplicas, - MaxReplicas: hpa.MaxReplicas, - Metrics: hpa.Metrics, - Behavior: hpa.Behavior, - } - params.HPASpec = hpaSpec - return params -} - -// GetSCCParams return params to render SCC from templates -func (n *NemoDatastoreV2) GetSCCParams() *rendertypes.SCCParams { - params := &rendertypes.SCCParams{} - // Set metadata - params.Name = "nemo-datastore-scc" - - params.ServiceAccountName = n.GetServiceAccountName() - return params -} - -// GetServiceMonitorParams return params to render Service Monitor from templates -func (n *NemoDatastoreV2) GetServiceMonitorParams() *rendertypes.ServiceMonitorParams { - params := &rendertypes.ServiceMonitorParams{} - serviceMonitor := n.GetServiceMonitor() - params.Enabled = n.IsServiceMonitorEnabled() - params.Name = n.GetName() - params.Namespace = n.GetNamespace() - svcLabels := n.GetServiceLabels() - maps.Copy(svcLabels, serviceMonitor.AdditionalLabels) - params.Labels = svcLabels - params.Annotations = n.GetServiceMonitorAnnotations() - - // Set Service Monitor spec - smSpec := monitoringv1.ServiceMonitorSpec{ - NamespaceSelector: monitoringv1.NamespaceSelector{MatchNames: []string{n.Namespace}}, - Selector: metav1.LabelSelector{MatchLabels: n.GetServiceLabels()}, - Endpoints: []monitoringv1.Endpoint{{Port: "service-port", ScrapeTimeout: serviceMonitor.ScrapeTimeout, Interval: serviceMonitor.Interval}}, - } - params.SMSpec = smSpec - return params -} - -func (n *NemoDatastoreV2) GetIngressAnnotations() map[string]string { - NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() - - if n.Spec.Expose.Ingress.Annotations != nil { - return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Expose.Ingress.Annotations) - } - return NemoDatastoreV2Annotations -} - -func (n *NemoDatastoreV2) GetServiceAnnotations() map[string]string { - NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() - - if n.Spec.Expose.Service.Annotations != nil { - return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Expose.Service.Annotations) - } - return NemoDatastoreV2Annotations -} - -func (n *NemoDatastoreV2) GetHPAAnnotations() map[string]string { - NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() - - if n.Spec.Scale.Annotations != nil { - return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Scale.Annotations) - } - return NemoDatastoreV2Annotations -} - -func (n *NemoDatastoreV2) GetServiceMonitorAnnotations() map[string]string { - NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() - - if n.Spec.Metrics.ServiceMonitor.Annotations != nil { - return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Metrics.ServiceMonitor.Annotations) - } - return NemoDatastoreV2Annotations -} - -func init() { - SchemeBuilder.Register(&NemoDatastoreV2{}, &NemoDatastoreV2List{}) -} diff --git a/api/apps/v1alpha1/zz_generated.deepcopy.go b/api/apps/v1alpha1/zz_generated.deepcopy.go index 12d4a6a5..d67b4c0b 100644 --- a/api/apps/v1alpha1/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/zz_generated.deepcopy.go @@ -983,26 +983,6 @@ func (in *NIMSource) DeepCopy() *NIMSource { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDataStoreParams) DeepCopyInto(out *NemoDataStoreParams) { - *out = *in - if in.InitContainerCommand != nil { - in, out := &in.InitContainerCommand, &out.InitContainerCommand - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDataStoreParams. -func (in *NemoDataStoreParams) DeepCopy() *NemoDataStoreParams { - if in == nil { - return nil - } - out := new(NemoDataStoreParams) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NemoDatastore) DeepCopyInto(out *NemoDatastore) { *out = *in @@ -1063,196 +1043,27 @@ func (in *NemoDatastoreList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreSpec) DeepCopyInto(out *NemoDatastoreSpec) { - *out = *in - in.Image.DeepCopyInto(&out.Image) - if in.Command != nil { - in, out := &in.Command, &out.Command - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Args != nil { - in, out := &in.Args, &out.Args - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Env != nil { - in, out := &in.Env, &out.Env - *out = make([]v1.EnvVar, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.Annotations != nil { - in, out := &in.Annotations, &out.Annotations - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.Tolerations != nil { - in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.PodAffinity != nil { - in, out := &in.PodAffinity, &out.PodAffinity - *out = new(v1.PodAffinity) - (*in).DeepCopyInto(*out) - } - if in.Resources != nil { - in, out := &in.Resources, &out.Resources - *out = new(v1.ResourceRequirements) - (*in).DeepCopyInto(*out) - } - in.Expose.DeepCopyInto(&out.Expose) - in.LivenessProbe.DeepCopyInto(&out.LivenessProbe) - in.ReadinessProbe.DeepCopyInto(&out.ReadinessProbe) - in.StartupProbe.DeepCopyInto(&out.StartupProbe) - in.Scale.DeepCopyInto(&out.Scale) - in.Metrics.DeepCopyInto(&out.Metrics) - if in.UserID != nil { - in, out := &in.UserID, &out.UserID - *out = new(int64) - **out = **in - } - if in.GroupID != nil { - in, out := &in.GroupID, &out.GroupID - *out = new(int64) - **out = **in - } - in.DataStoreParams.DeepCopyInto(&out.DataStoreParams) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreSpec. -func (in *NemoDatastoreSpec) DeepCopy() *NemoDatastoreSpec { - if in == nil { - return nil - } - out := new(NemoDatastoreSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreStatus) DeepCopyInto(out *NemoDatastoreStatus) { - *out = *in - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]metav1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreStatus. -func (in *NemoDatastoreStatus) DeepCopy() *NemoDatastoreStatus { - if in == nil { - return nil - } - out := new(NemoDatastoreStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreV2) DeepCopyInto(out *NemoDatastoreV2) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2. -func (in *NemoDatastoreV2) DeepCopy() *NemoDatastoreV2 { - if in == nil { - return nil - } - out := new(NemoDatastoreV2) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *NemoDatastoreV2) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreV2List) DeepCopyInto(out *NemoDatastoreV2List) { +func (in *NemoDatastoreParams) DeepCopyInto(out *NemoDatastoreParams) { *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]NemoDatastoreV2, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2List. -func (in *NemoDatastoreV2List) DeepCopy() *NemoDatastoreV2List { - if in == nil { - return nil - } - out := new(NemoDatastoreV2List) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *NemoDatastoreV2List) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreV2Params) DeepCopyInto(out *NemoDatastoreV2Params) { - *out = *in - if in.Persistence != nil { - in, out := &in.Persistence, &out.Persistence - *out = new(Persistence) + if in.PVC != nil { + in, out := &in.PVC, &out.PVC + *out = new(PersistentVolumeClaim) (*in).DeepCopyInto(*out) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Params. -func (in *NemoDatastoreV2Params) DeepCopy() *NemoDatastoreV2Params { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreParams. +func (in *NemoDatastoreParams) DeepCopy() *NemoDatastoreParams { if in == nil { return nil } - out := new(NemoDatastoreV2Params) + out := new(NemoDatastoreParams) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreV2Spec) DeepCopyInto(out *NemoDatastoreV2Spec) { +func (in *NemoDatastoreSpec) DeepCopyInto(out *NemoDatastoreSpec) { *out = *in in.Image.DeepCopyInto(&out.Image) if in.Command != nil { @@ -1329,18 +1140,18 @@ func (in *NemoDatastoreV2Spec) DeepCopyInto(out *NemoDatastoreV2Spec) { in.DataStoreParams.DeepCopyInto(&out.DataStoreParams) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Spec. -func (in *NemoDatastoreV2Spec) DeepCopy() *NemoDatastoreV2Spec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreSpec. +func (in *NemoDatastoreSpec) DeepCopy() *NemoDatastoreSpec { if in == nil { return nil } - out := new(NemoDatastoreV2Spec) + out := new(NemoDatastoreSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreV2Status) DeepCopyInto(out *NemoDatastoreV2Status) { +func (in *NemoDatastoreStatus) DeepCopyInto(out *NemoDatastoreStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions @@ -1351,12 +1162,12 @@ func (in *NemoDatastoreV2Status) DeepCopyInto(out *NemoDatastoreV2Status) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Status. -func (in *NemoDatastoreV2Status) DeepCopy() *NemoDatastoreV2Status { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreStatus. +func (in *NemoDatastoreStatus) DeepCopy() *NemoDatastoreStatus { if in == nil { return nil } - out := new(NemoDatastoreV2Status) + out := new(NemoDatastoreStatus) in.DeepCopyInto(out) return out } @@ -1891,38 +1702,6 @@ func (in *NemoGuardrailStatus) DeepCopy() *NemoGuardrailStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Persistence) DeepCopyInto(out *Persistence) { - *out = *in - if in.AccessModes != nil { - in, out := &in.AccessModes, &out.AccessModes - *out = make([]v1.PersistentVolumeAccessMode, len(*in)) - copy(*out, *in) - } - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.StorageClass != nil { - in, out := &in.StorageClass, &out.StorageClass - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Persistence. -func (in *Persistence) DeepCopy() *Persistence { - if in == nil { - return nil - } - out := new(Persistence) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PersistentVolumeClaim) DeepCopyInto(out *PersistentVolumeClaim) { *out = *in diff --git a/api/versioned/typed/apps/v1alpha1/apps_client.go b/api/versioned/typed/apps/v1alpha1/apps_client.go index b6f2e4ca..045f4265 100644 --- a/api/versioned/typed/apps/v1alpha1/apps_client.go +++ b/api/versioned/typed/apps/v1alpha1/apps_client.go @@ -31,7 +31,6 @@ type AppsV1alpha1Interface interface { NIMPipelinesGetter NIMServicesGetter NemoDatastoresGetter - NemoDatastoreV2sGetter NemoEntitystoresGetter NemoEvaluatorsGetter NemoGuardrailsGetter @@ -58,10 +57,6 @@ func (c *AppsV1alpha1Client) NemoDatastores(namespace string) NemoDatastoreInter return newNemoDatastores(c, namespace) } -func (c *AppsV1alpha1Client) NemoDatastoreV2s(namespace string) NemoDatastoreV2Interface { - return newNemoDatastoreV2s(c, namespace) -} - func (c *AppsV1alpha1Client) NemoEntitystores(namespace string) NemoEntitystoreInterface { return newNemoEntitystores(c, namespace) } diff --git a/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go b/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go index 9809e22c..1896f204 100644 --- a/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go +++ b/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go @@ -43,10 +43,6 @@ func (c *FakeAppsV1alpha1) NemoDatastores(namespace string) v1alpha1.NemoDatasto return &FakeNemoDatastores{c, namespace} } -func (c *FakeAppsV1alpha1) NemoDatastoreV2s(namespace string) v1alpha1.NemoDatastoreV2Interface { - return &FakeNemoDatastoreV2s{c, namespace} -} - func (c *FakeAppsV1alpha1) NemoEntitystores(namespace string) v1alpha1.NemoEntitystoreInterface { return &FakeNemoEntitystores{c, namespace} } diff --git a/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go b/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go deleted file mode 100644 index fd957d51..00000000 --- a/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - v1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" -) - -// FakeNemoDatastoreV2s implements NemoDatastoreV2Interface -type FakeNemoDatastoreV2s struct { - Fake *FakeAppsV1alpha1 - ns string -} - -var nemodatastorev2sResource = v1alpha1.SchemeGroupVersion.WithResource("nemodatastorev2s") - -var nemodatastorev2sKind = v1alpha1.SchemeGroupVersion.WithKind("NemoDatastoreV2") - -// Get takes name of the nemoDatastoreV2, and returns the corresponding nemoDatastoreV2 object, and an error if there is any. -func (c *FakeNemoDatastoreV2s) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.NemoDatastoreV2, err error) { - emptyResult := &v1alpha1.NemoDatastoreV2{} - obj, err := c.Fake. - Invokes(testing.NewGetActionWithOptions(nemodatastorev2sResource, c.ns, name, options), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1alpha1.NemoDatastoreV2), err -} - -// List takes label and field selectors, and returns the list of NemoDatastoreV2s that match those selectors. -func (c *FakeNemoDatastoreV2s) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.NemoDatastoreV2List, err error) { - emptyResult := &v1alpha1.NemoDatastoreV2List{} - obj, err := c.Fake. - Invokes(testing.NewListActionWithOptions(nemodatastorev2sResource, nemodatastorev2sKind, c.ns, opts), emptyResult) - - if obj == nil { - return emptyResult, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &v1alpha1.NemoDatastoreV2List{ListMeta: obj.(*v1alpha1.NemoDatastoreV2List).ListMeta} - for _, item := range obj.(*v1alpha1.NemoDatastoreV2List).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested nemoDatastoreV2s. -func (c *FakeNemoDatastoreV2s) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchActionWithOptions(nemodatastorev2sResource, c.ns, opts)) - -} - -// Create takes the representation of a nemoDatastoreV2 and creates it. Returns the server's representation of the nemoDatastoreV2, and an error, if there is any. -func (c *FakeNemoDatastoreV2s) Create(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.CreateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { - emptyResult := &v1alpha1.NemoDatastoreV2{} - obj, err := c.Fake. - Invokes(testing.NewCreateActionWithOptions(nemodatastorev2sResource, c.ns, nemoDatastoreV2, opts), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1alpha1.NemoDatastoreV2), err -} - -// Update takes the representation of a nemoDatastoreV2 and updates it. Returns the server's representation of the nemoDatastoreV2, and an error, if there is any. -func (c *FakeNemoDatastoreV2s) Update(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { - emptyResult := &v1alpha1.NemoDatastoreV2{} - obj, err := c.Fake. - Invokes(testing.NewUpdateActionWithOptions(nemodatastorev2sResource, c.ns, nemoDatastoreV2, opts), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1alpha1.NemoDatastoreV2), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeNemoDatastoreV2s) UpdateStatus(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { - emptyResult := &v1alpha1.NemoDatastoreV2{} - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceActionWithOptions(nemodatastorev2sResource, "status", c.ns, nemoDatastoreV2, opts), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1alpha1.NemoDatastoreV2), err -} - -// Delete takes name of the nemoDatastoreV2 and deletes it. Returns an error if one occurs. -func (c *FakeNemoDatastoreV2s) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteActionWithOptions(nemodatastorev2sResource, c.ns, name, opts), &v1alpha1.NemoDatastoreV2{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeNemoDatastoreV2s) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionActionWithOptions(nemodatastorev2sResource, c.ns, opts, listOpts) - - _, err := c.Fake.Invokes(action, &v1alpha1.NemoDatastoreV2List{}) - return err -} - -// Patch applies the patch and returns the patched nemoDatastoreV2. -func (c *FakeNemoDatastoreV2s) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NemoDatastoreV2, err error) { - emptyResult := &v1alpha1.NemoDatastoreV2{} - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceActionWithOptions(nemodatastorev2sResource, c.ns, name, pt, data, opts, subresources...), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1alpha1.NemoDatastoreV2), err -} diff --git a/api/versioned/typed/apps/v1alpha1/generated_expansion.go b/api/versioned/typed/apps/v1alpha1/generated_expansion.go index 59602980..78164a8c 100644 --- a/api/versioned/typed/apps/v1alpha1/generated_expansion.go +++ b/api/versioned/typed/apps/v1alpha1/generated_expansion.go @@ -25,8 +25,6 @@ type NIMServiceExpansion interface{} type NemoDatastoreExpansion interface{} -type NemoDatastoreV2Expansion interface{} - type NemoEntitystoreExpansion interface{} type NemoEvaluatorExpansion interface{} diff --git a/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go b/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go deleted file mode 100644 index d58def11..00000000 --- a/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - - v1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" - scheme "github.com/NVIDIA/k8s-nim-operator/api/versioned/scheme" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - gentype "k8s.io/client-go/gentype" -) - -// NemoDatastoreV2sGetter has a method to return a NemoDatastoreV2Interface. -// A group's client should implement this interface. -type NemoDatastoreV2sGetter interface { - NemoDatastoreV2s(namespace string) NemoDatastoreV2Interface -} - -// NemoDatastoreV2Interface has methods to work with NemoDatastoreV2 resources. -type NemoDatastoreV2Interface interface { - Create(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.CreateOptions) (*v1alpha1.NemoDatastoreV2, error) - Update(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (*v1alpha1.NemoDatastoreV2, error) - // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). - UpdateStatus(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (*v1alpha1.NemoDatastoreV2, error) - Delete(ctx context.Context, name string, opts v1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.NemoDatastoreV2, error) - List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.NemoDatastoreV2List, error) - Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NemoDatastoreV2, err error) - NemoDatastoreV2Expansion -} - -// nemoDatastoreV2s implements NemoDatastoreV2Interface -type nemoDatastoreV2s struct { - *gentype.ClientWithList[*v1alpha1.NemoDatastoreV2, *v1alpha1.NemoDatastoreV2List] -} - -// newNemoDatastoreV2s returns a NemoDatastoreV2s -func newNemoDatastoreV2s(c *AppsV1alpha1Client, namespace string) *nemoDatastoreV2s { - return &nemoDatastoreV2s{ - gentype.NewClientWithList[*v1alpha1.NemoDatastoreV2, *v1alpha1.NemoDatastoreV2List]( - "nemodatastorev2s", - c.RESTClient(), - scheme.ParameterCodec, - namespace, - func() *v1alpha1.NemoDatastoreV2 { return &v1alpha1.NemoDatastoreV2{} }, - func() *v1alpha1.NemoDatastoreV2List { return &v1alpha1.NemoDatastoreV2List{} }), - } -} diff --git a/bundle/manifests/apps.nvidia.com_nemodatastores.yaml b/bundle/manifests/apps.nvidia.com_nemodatastores.yaml index ffdc4923..ae998d7d 100644 --- a/bundle/manifests/apps.nvidia.com_nemodatastores.yaml +++ b/bundle/manifests/apps.nvidia.com_nemodatastores.yaml @@ -65,40 +65,60 @@ spec: type: array dataStoreParams: properties: - appVersion: + datastoreConfigSecret: type: string - databaseHost: + datastoreInitSecret: type: string - databasePort: + datastoreInlineConfigSecret: type: string - databaseURL: + datastoreSettingsSecret: type: string dbSecret: type: string - envConfigmap: + giteaAdminSecret: type: string - envSecret: + lfsJwtSecret: type: string - giteaEndpoint: - type: string - giteaSecret: - type: string - initContainerCommand: - items: - type: string - type: array - initContainerImage: + objStoreSecret: type: string + persistence: + properties: + accessModes: + items: + type: string + type: array + claimName: + type: string + create: + type: boolean + enabled: + type: boolean + labels: + additionalProperties: + type: string + type: object + size: + type: string + storageClass: + type: string + required: + - claimName + - create + - enabled + - size + type: object + sshEnabled: + type: boolean required: - - appVersion - - databaseHost - - databasePort - - databaseURL + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret - dbSecret - - envConfigmap - - envSecret - - giteaEndpoint - - giteaSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - sshEnabled type: object env: items: diff --git a/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml b/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml deleted file mode 100644 index 518d4f75..00000000 --- a/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml +++ /dev/null @@ -1,2230 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.2 - name: nemodatastorev2s.apps.nvidia.com -spec: - group: apps.nvidia.com - names: - kind: NemoDatastoreV2 - listKind: NemoDatastoreV2List - plural: nemodatastorev2s - singular: nemodatastorev2 - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.state - name: Status - type: string - - format: date-time - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 - properties: - annotations: - additionalProperties: - type: string - type: object - args: - items: - type: string - type: array - authSecret: - description: The name of an secret that contains authn for the NGC - NIM service API - type: string - command: - items: - type: string - type: array - dataStoreParams: - properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: - properties: - accessModes: - items: - type: string - type: array - claimName: - type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: - type: string - storageClass: - type: string - required: - - claimName - - create - - enabled - - size - type: object - sshEnabled: - type: boolean - required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled - type: object - env: - items: - description: EnvVar represents an environment variable present in - a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot - be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is - written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified - API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed - resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - expose: - description: Expose defines attributes to expose the service - properties: - ingress: - description: Ingress defines attributes to enable ingress for - the service - properties: - annotations: - additionalProperties: - type: string - type: object - enabled: - description: ingress, or virtualService - not both - type: boolean - spec: - description: IngressSpec describes the Ingress the user wishes - to exist. - properties: - defaultBackend: - description: |- - defaultBackend is the backend that should handle requests that don't - match any rule. If Rules are not specified, DefaultBackend must be specified. - If DefaultBackend is not set, the handling of requests that do not match any - of the rules will be up to the Ingress controller. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - ingressClassName: - description: |- - ingressClassName is the name of an IngressClass cluster resource. Ingress - controller implementations use this field to know whether they should be - serving this Ingress resource, by a transitive connection - (controller -> IngressClass -> Ingress resource). Although the - `kubernetes.io/ingress.class` annotation (simple constant name) was never - formally defined, it was widely supported by Ingress controllers to create - a direct binding between Ingress controller and Ingress resources. Newly - created Ingress resources should prefer using the field. However, even - though the annotation is officially deprecated, for backwards compatibility - reasons, ingress controllers should still honor that annotation if present. - type: string - rules: - description: |- - rules is a list of host rules used to configure the Ingress. If unspecified, - or no rule matches, all traffic is sent to the default backend. - items: - description: |- - IngressRule represents the rules mapping the paths under a specified host to - the related backend services. Incoming requests are first evaluated for a host - match, then routed to the backend associated with the matching IngressRuleValue. - properties: - host: - description: "host is the fully qualified domain - name of a network host, as defined by RFC 3986.\nNote - the following deviations from the \"host\" part - of the\nURI as defined in RFC 3986:\n1. IPs are - not allowed. Currently an IngressRuleValue can - only apply to\n the IP in the Spec of the parent - Ingress.\n2. The `:` delimiter is not respected - because ports are not allowed.\n\t Currently - the port of an Ingress is implicitly :80 for http - and\n\t :443 for https.\nBoth these may change - in the future.\nIncoming requests are matched - against the host before the\nIngressRuleValue. - If the host is unspecified, the Ingress routes - all\ntraffic based on the specified IngressRuleValue.\n\nhost - can be \"precise\" which is a domain name without - the terminating dot of\na network host (e.g. \"foo.bar.com\") - or \"wildcard\", which is a domain name\nprefixed - with a single wildcard label (e.g. \"*.foo.com\").\nThe - wildcard character '*' must appear by itself as - the first DNS label and\nmatches only a single - label. You cannot have a wildcard label by itself - (e.g. Host == \"*\").\nRequests will be matched - against the Host field in the following way:\n1. - If host is precise, the request matches this rule - if the http host header is equal to Host.\n2. - If host is a wildcard, then the request matches - this rule if the http host header\nis to equal - to the suffix (removing the first label) of the - wildcard rule." - type: string - http: - description: |- - HTTPIngressRuleValue is a list of http selectors pointing to backends. - In the example: http:///? -> backend where - where parts of the url correspond to RFC 3986, this resource will be used - to match against everything after the last '/' and before the first '?' - or '#'. - properties: - paths: - description: paths is a collection of paths - that map requests to backends. - items: - description: |- - HTTPIngressPath associates a path with a backend. Incoming urls matching the - path are forwarded to the backend. - properties: - backend: - description: |- - backend defines the referenced service endpoint to which the traffic - will be forwarded to. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - path: - description: |- - path is matched against the path of an incoming request. Currently it can - contain characters disallowed from the conventional "path" part of a URL - as defined by RFC 3986. Paths must begin with a '/' and must be present - when using PathType with value "Exact" or "Prefix". - type: string - pathType: - description: |- - pathType determines the interpretation of the path matching. PathType can - be one of the following values: - * Exact: Matches the URL path exactly. - * Prefix: Matches based on a URL path prefix split by '/'. Matching is - done on a path element by element basis. A path element refers is the - list of labels in the path split by the '/' separator. A request is a - match for path p if every p is an element-wise prefix of p of the - request path. Note that if the last element of the path is a substring - of the last element in request path, it is not a match (e.g. /foo/bar - matches /foo/bar/baz, but does not match /foo/barbaz). - * ImplementationSpecific: Interpretation of the Path matching is up to - the IngressClass. Implementations can treat this as a separate PathType - or treat it identically to Prefix or Exact path types. - Implementations are required to support all path types. - type: string - required: - - backend - - pathType - type: object - type: array - x-kubernetes-list-type: atomic - required: - - paths - type: object - type: object - type: array - x-kubernetes-list-type: atomic - tls: - description: |- - tls represents the TLS configuration. Currently the Ingress only supports a - single TLS port, 443. If multiple members of this list specify different hosts, - they will be multiplexed on the same port according to the hostname specified - through the SNI TLS extension, if the ingress controller fulfilling the - ingress supports SNI. - items: - description: IngressTLS describes the transport layer - security associated with an ingress. - properties: - hosts: - description: |- - hosts is a list of hosts included in the TLS certificate. The values in - this list must match the name/s used in the tlsSecret. Defaults to the - wildcard host setting for the loadbalancer controller fulfilling this - Ingress, if left unspecified. - items: - type: string - type: array - x-kubernetes-list-type: atomic - secretName: - description: |- - secretName is the name of the secret used to terminate TLS traffic on - port 443. Field is left optional to allow TLS routing based on SNI - hostname alone. If the SNI host in a listener conflicts with the "Host" - header field used by an IngressRule, the SNI host is used for termination - and value of the "Host" header is used for routing. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - service: - description: Service defines attributes to create a service - properties: - annotations: - additionalProperties: - type: string - type: object - name: - description: override the default service name - type: string - port: - default: 8000 - format: int32 - type: integer - type: - description: Service Type string describes ingress methods - for a service - type: string - required: - - port - type: object - type: object - groupID: - format: int64 - type: integer - image: - description: Image defines image attributes - properties: - pullPolicy: - type: string - pullSecrets: - items: - type: string - type: array - repository: - type: string - tag: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - livenessProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - metrics: - description: Metrics defines attributes to setup metrics collection - properties: - enabled: - type: boolean - serviceMonitor: - description: for use with the Prometheus Operator and the primary - service object - properties: - additionalLabels: - additionalProperties: - type: string - type: object - annotations: - additionalProperties: - type: string - type: object - interval: - description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - scrapeTimeout: - description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - type: object - type: object - nodeSelector: - additionalProperties: - type: string - type: object - podAffinity: - description: Pod affinity is a group of inter pod affinity scheduling - rules. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with - the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - readinessProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - replicas: - default: 1 - minimum: 1 - type: integer - resources: - description: ResourceRequirements describes the compute resource requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - runtimeClass: - type: string - scale: - description: Autoscaling defines attributes to automatically scale - the service based on metrics - properties: - annotations: - additionalProperties: - type: string - type: object - enabled: - type: boolean - hpa: - description: HorizontalPodAutoscalerSpec defines the parameters - required to setup HPA - properties: - behavior: - description: |- - HorizontalPodAutoscalerBehavior configures the scaling behavior of the target - in both Up and Down directions (scaleUp and scaleDown fields respectively). - properties: - scaleDown: - description: |- - scaleDown is scaling policy for scaling Down. - If not set, the default value is to allow to scale down to minReplicas pods, with a - 300 second stabilization window (i.e., the highest recommendation for - the last 300sec is used). - properties: - policies: - description: |- - policies is a list of potential scaling polices which can be used during scaling. - At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid - items: - description: HPAScalingPolicy is a single policy - which must hold true for a specified past interval. - properties: - periodSeconds: - description: |- - periodSeconds specifies the window of time for which the policy should hold true. - PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). - format: int32 - type: integer - type: - description: type is used to specify the scaling - policy. - type: string - value: - description: |- - value contains the amount of change which is permitted by the policy. - It must be greater than zero - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - x-kubernetes-list-type: atomic - selectPolicy: - description: |- - selectPolicy is used to specify which policy should be used. - If not set, the default value Max is used. - type: string - stabilizationWindowSeconds: - description: |- - stabilizationWindowSeconds is the number of seconds for which past recommendations should be - considered while scaling up or scaling down. - StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). - If not set, use the default values: - - For scale up: 0 (i.e. no stabilization is done). - - For scale down: 300 (i.e. the stabilization window is 300 seconds long). - format: int32 - type: integer - type: object - scaleUp: - description: |- - scaleUp is scaling policy for scaling Up. - If not set, the default value is the higher of: - * increase no more than 4 pods per 60 seconds - * double the number of pods per 60 seconds - No stabilization is used. - properties: - policies: - description: |- - policies is a list of potential scaling polices which can be used during scaling. - At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid - items: - description: HPAScalingPolicy is a single policy - which must hold true for a specified past interval. - properties: - periodSeconds: - description: |- - periodSeconds specifies the window of time for which the policy should hold true. - PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). - format: int32 - type: integer - type: - description: type is used to specify the scaling - policy. - type: string - value: - description: |- - value contains the amount of change which is permitted by the policy. - It must be greater than zero - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - x-kubernetes-list-type: atomic - selectPolicy: - description: |- - selectPolicy is used to specify which policy should be used. - If not set, the default value Max is used. - type: string - stabilizationWindowSeconds: - description: |- - stabilizationWindowSeconds is the number of seconds for which past recommendations should be - considered while scaling up or scaling down. - StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). - If not set, use the default values: - - For scale up: 0 (i.e. no stabilization is done). - - For scale down: 300 (i.e. the stabilization window is 300 seconds long). - format: int32 - type: integer - type: object - type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: - description: |- - MetricSpec specifies how to scale based on a single metric - (only `type` and one other matching field should be set at once). - properties: - containerResource: - description: |- - containerResource refers to a resource metric (such as those specified in - requests and limits) known to Kubernetes describing a single container in - each pod of the current scale target (e.g. CPU or memory). Such metrics are - built in to Kubernetes, and have special scaling options on top of those - available to normal per-pod metrics using the "pods" source. - This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. - properties: - container: - description: container is the name of the container - in the pods of the scaling target - type: string - name: - description: name is the name of the resource in - question. - type: string - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - container - - name - - target - type: object - external: - description: |- - external refers to a global metric that is not associated - with any Kubernetes object. It allows autoscaling based on information - coming from components running outside of cluster - (for example length of queue in cloud messaging service, or - QPS from loadbalancer running outside of cluster). - properties: - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - metric - - target - type: object - object: - description: |- - object refers to a metric describing a single kubernetes object - (for example, hits-per-second on an Ingress object). - properties: - describedObject: - description: describedObject specifies the descriptions - of a object,such as kind,name apiVersion - properties: - apiVersion: - description: apiVersion is the API version of - the referent - type: string - kind: - description: 'kind is the kind of the referent; - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'name is the name of the referent; - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - required: - - kind - - name - type: object - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - describedObject - - metric - - target - type: object - pods: - description: |- - pods refers to a metric describing each pod in the current scale target - (for example, transactions-processed-per-second). The values will be - averaged together before being compared to the target value. - properties: - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - metric - - target - type: object - resource: - description: |- - resource refers to a resource metric (such as those specified in - requests and limits) known to Kubernetes describing each pod in the - current scale target (e.g. CPU or memory). Such metrics are built in to - Kubernetes, and have special scaling options on top of those available - to normal per-pod metrics using the "pods" source. - properties: - name: - description: name is the name of the resource in - question. - type: string - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - name - - target - type: object - type: - description: |- - type is the type of metric source. It should be one of "ContainerResource", "External", - "Object", "Pods" or "Resource", each mapping to a matching field in the object. - Note: "ContainerResource" type is available on when the feature-gate - HPAContainerMetrics is enabled - type: string - required: - - type - type: object - type: array - minReplicas: - format: int32 - type: integer - required: - - maxReplicas - type: object - type: object - startupProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - userID: - format: int64 - type: integer - required: - - authSecret - - dataStoreParams - type: object - status: - description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 - properties: - availableReplicas: - format: int32 - type: integer - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - state: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/cmd/main.go b/cmd/main.go index d0fb4d33..8b626558 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -188,17 +188,6 @@ func main() { os.Exit(1) } - if err = controller.NewNemoDatastoreReconciler( - mgr.GetClient(), - mgr.GetScheme(), - updater, - render.NewRenderer("/manifests"), - ctrl.Log.WithName("controllers").WithName("NemoDatastore"), - ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NemoDatastore") - os.Exit(1) - } - if err = controller.NewNemoEvaluatorReconciler( mgr.GetClient(), mgr.GetScheme(), @@ -221,14 +210,14 @@ func main() { os.Exit(1) } - if err = controller.NewNemoDatastoreV2Reconciler( + if err = controller.NewNemoDatastoreReconciler( mgr.GetClient(), mgr.GetScheme(), updater, render.NewRenderer("/manifests"), - ctrl.Log.WithName("controllers").WithName("NemoDatastoreV2"), + ctrl.Log.WithName("controllers").WithName("NemoDatastore"), ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NemoDatastoreV2") + setupLog.Error(err, "unable to create controller", "controller", "NemoDatastore") os.Exit(1) } diff --git a/config/crd/bases/apps.nvidia.com_nemodatastores.yaml b/config/crd/bases/apps.nvidia.com_nemodatastores.yaml index ffdc4923..ae998d7d 100644 --- a/config/crd/bases/apps.nvidia.com_nemodatastores.yaml +++ b/config/crd/bases/apps.nvidia.com_nemodatastores.yaml @@ -65,40 +65,60 @@ spec: type: array dataStoreParams: properties: - appVersion: + datastoreConfigSecret: type: string - databaseHost: + datastoreInitSecret: type: string - databasePort: + datastoreInlineConfigSecret: type: string - databaseURL: + datastoreSettingsSecret: type: string dbSecret: type: string - envConfigmap: + giteaAdminSecret: type: string - envSecret: + lfsJwtSecret: type: string - giteaEndpoint: - type: string - giteaSecret: - type: string - initContainerCommand: - items: - type: string - type: array - initContainerImage: + objStoreSecret: type: string + persistence: + properties: + accessModes: + items: + type: string + type: array + claimName: + type: string + create: + type: boolean + enabled: + type: boolean + labels: + additionalProperties: + type: string + type: object + size: + type: string + storageClass: + type: string + required: + - claimName + - create + - enabled + - size + type: object + sshEnabled: + type: boolean required: - - appVersion - - databaseHost - - databasePort - - databaseURL + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret - dbSecret - - envConfigmap - - envSecret - - giteaEndpoint - - giteaSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - sshEnabled type: object env: items: diff --git a/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml b/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml deleted file mode 100644 index 518d4f75..00000000 --- a/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml +++ /dev/null @@ -1,2230 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.2 - name: nemodatastorev2s.apps.nvidia.com -spec: - group: apps.nvidia.com - names: - kind: NemoDatastoreV2 - listKind: NemoDatastoreV2List - plural: nemodatastorev2s - singular: nemodatastorev2 - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.state - name: Status - type: string - - format: date-time - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 - properties: - annotations: - additionalProperties: - type: string - type: object - args: - items: - type: string - type: array - authSecret: - description: The name of an secret that contains authn for the NGC - NIM service API - type: string - command: - items: - type: string - type: array - dataStoreParams: - properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: - properties: - accessModes: - items: - type: string - type: array - claimName: - type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: - type: string - storageClass: - type: string - required: - - claimName - - create - - enabled - - size - type: object - sshEnabled: - type: boolean - required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled - type: object - env: - items: - description: EnvVar represents an environment variable present in - a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot - be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is - written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified - API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed - resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - expose: - description: Expose defines attributes to expose the service - properties: - ingress: - description: Ingress defines attributes to enable ingress for - the service - properties: - annotations: - additionalProperties: - type: string - type: object - enabled: - description: ingress, or virtualService - not both - type: boolean - spec: - description: IngressSpec describes the Ingress the user wishes - to exist. - properties: - defaultBackend: - description: |- - defaultBackend is the backend that should handle requests that don't - match any rule. If Rules are not specified, DefaultBackend must be specified. - If DefaultBackend is not set, the handling of requests that do not match any - of the rules will be up to the Ingress controller. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - ingressClassName: - description: |- - ingressClassName is the name of an IngressClass cluster resource. Ingress - controller implementations use this field to know whether they should be - serving this Ingress resource, by a transitive connection - (controller -> IngressClass -> Ingress resource). Although the - `kubernetes.io/ingress.class` annotation (simple constant name) was never - formally defined, it was widely supported by Ingress controllers to create - a direct binding between Ingress controller and Ingress resources. Newly - created Ingress resources should prefer using the field. However, even - though the annotation is officially deprecated, for backwards compatibility - reasons, ingress controllers should still honor that annotation if present. - type: string - rules: - description: |- - rules is a list of host rules used to configure the Ingress. If unspecified, - or no rule matches, all traffic is sent to the default backend. - items: - description: |- - IngressRule represents the rules mapping the paths under a specified host to - the related backend services. Incoming requests are first evaluated for a host - match, then routed to the backend associated with the matching IngressRuleValue. - properties: - host: - description: "host is the fully qualified domain - name of a network host, as defined by RFC 3986.\nNote - the following deviations from the \"host\" part - of the\nURI as defined in RFC 3986:\n1. IPs are - not allowed. Currently an IngressRuleValue can - only apply to\n the IP in the Spec of the parent - Ingress.\n2. The `:` delimiter is not respected - because ports are not allowed.\n\t Currently - the port of an Ingress is implicitly :80 for http - and\n\t :443 for https.\nBoth these may change - in the future.\nIncoming requests are matched - against the host before the\nIngressRuleValue. - If the host is unspecified, the Ingress routes - all\ntraffic based on the specified IngressRuleValue.\n\nhost - can be \"precise\" which is a domain name without - the terminating dot of\na network host (e.g. \"foo.bar.com\") - or \"wildcard\", which is a domain name\nprefixed - with a single wildcard label (e.g. \"*.foo.com\").\nThe - wildcard character '*' must appear by itself as - the first DNS label and\nmatches only a single - label. You cannot have a wildcard label by itself - (e.g. Host == \"*\").\nRequests will be matched - against the Host field in the following way:\n1. - If host is precise, the request matches this rule - if the http host header is equal to Host.\n2. - If host is a wildcard, then the request matches - this rule if the http host header\nis to equal - to the suffix (removing the first label) of the - wildcard rule." - type: string - http: - description: |- - HTTPIngressRuleValue is a list of http selectors pointing to backends. - In the example: http:///? -> backend where - where parts of the url correspond to RFC 3986, this resource will be used - to match against everything after the last '/' and before the first '?' - or '#'. - properties: - paths: - description: paths is a collection of paths - that map requests to backends. - items: - description: |- - HTTPIngressPath associates a path with a backend. Incoming urls matching the - path are forwarded to the backend. - properties: - backend: - description: |- - backend defines the referenced service endpoint to which the traffic - will be forwarded to. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - path: - description: |- - path is matched against the path of an incoming request. Currently it can - contain characters disallowed from the conventional "path" part of a URL - as defined by RFC 3986. Paths must begin with a '/' and must be present - when using PathType with value "Exact" or "Prefix". - type: string - pathType: - description: |- - pathType determines the interpretation of the path matching. PathType can - be one of the following values: - * Exact: Matches the URL path exactly. - * Prefix: Matches based on a URL path prefix split by '/'. Matching is - done on a path element by element basis. A path element refers is the - list of labels in the path split by the '/' separator. A request is a - match for path p if every p is an element-wise prefix of p of the - request path. Note that if the last element of the path is a substring - of the last element in request path, it is not a match (e.g. /foo/bar - matches /foo/bar/baz, but does not match /foo/barbaz). - * ImplementationSpecific: Interpretation of the Path matching is up to - the IngressClass. Implementations can treat this as a separate PathType - or treat it identically to Prefix or Exact path types. - Implementations are required to support all path types. - type: string - required: - - backend - - pathType - type: object - type: array - x-kubernetes-list-type: atomic - required: - - paths - type: object - type: object - type: array - x-kubernetes-list-type: atomic - tls: - description: |- - tls represents the TLS configuration. Currently the Ingress only supports a - single TLS port, 443. If multiple members of this list specify different hosts, - they will be multiplexed on the same port according to the hostname specified - through the SNI TLS extension, if the ingress controller fulfilling the - ingress supports SNI. - items: - description: IngressTLS describes the transport layer - security associated with an ingress. - properties: - hosts: - description: |- - hosts is a list of hosts included in the TLS certificate. The values in - this list must match the name/s used in the tlsSecret. Defaults to the - wildcard host setting for the loadbalancer controller fulfilling this - Ingress, if left unspecified. - items: - type: string - type: array - x-kubernetes-list-type: atomic - secretName: - description: |- - secretName is the name of the secret used to terminate TLS traffic on - port 443. Field is left optional to allow TLS routing based on SNI - hostname alone. If the SNI host in a listener conflicts with the "Host" - header field used by an IngressRule, the SNI host is used for termination - and value of the "Host" header is used for routing. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - service: - description: Service defines attributes to create a service - properties: - annotations: - additionalProperties: - type: string - type: object - name: - description: override the default service name - type: string - port: - default: 8000 - format: int32 - type: integer - type: - description: Service Type string describes ingress methods - for a service - type: string - required: - - port - type: object - type: object - groupID: - format: int64 - type: integer - image: - description: Image defines image attributes - properties: - pullPolicy: - type: string - pullSecrets: - items: - type: string - type: array - repository: - type: string - tag: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - livenessProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - metrics: - description: Metrics defines attributes to setup metrics collection - properties: - enabled: - type: boolean - serviceMonitor: - description: for use with the Prometheus Operator and the primary - service object - properties: - additionalLabels: - additionalProperties: - type: string - type: object - annotations: - additionalProperties: - type: string - type: object - interval: - description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - scrapeTimeout: - description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - type: object - type: object - nodeSelector: - additionalProperties: - type: string - type: object - podAffinity: - description: Pod affinity is a group of inter pod affinity scheduling - rules. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with - the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - readinessProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - replicas: - default: 1 - minimum: 1 - type: integer - resources: - description: ResourceRequirements describes the compute resource requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - runtimeClass: - type: string - scale: - description: Autoscaling defines attributes to automatically scale - the service based on metrics - properties: - annotations: - additionalProperties: - type: string - type: object - enabled: - type: boolean - hpa: - description: HorizontalPodAutoscalerSpec defines the parameters - required to setup HPA - properties: - behavior: - description: |- - HorizontalPodAutoscalerBehavior configures the scaling behavior of the target - in both Up and Down directions (scaleUp and scaleDown fields respectively). - properties: - scaleDown: - description: |- - scaleDown is scaling policy for scaling Down. - If not set, the default value is to allow to scale down to minReplicas pods, with a - 300 second stabilization window (i.e., the highest recommendation for - the last 300sec is used). - properties: - policies: - description: |- - policies is a list of potential scaling polices which can be used during scaling. - At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid - items: - description: HPAScalingPolicy is a single policy - which must hold true for a specified past interval. - properties: - periodSeconds: - description: |- - periodSeconds specifies the window of time for which the policy should hold true. - PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). - format: int32 - type: integer - type: - description: type is used to specify the scaling - policy. - type: string - value: - description: |- - value contains the amount of change which is permitted by the policy. - It must be greater than zero - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - x-kubernetes-list-type: atomic - selectPolicy: - description: |- - selectPolicy is used to specify which policy should be used. - If not set, the default value Max is used. - type: string - stabilizationWindowSeconds: - description: |- - stabilizationWindowSeconds is the number of seconds for which past recommendations should be - considered while scaling up or scaling down. - StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). - If not set, use the default values: - - For scale up: 0 (i.e. no stabilization is done). - - For scale down: 300 (i.e. the stabilization window is 300 seconds long). - format: int32 - type: integer - type: object - scaleUp: - description: |- - scaleUp is scaling policy for scaling Up. - If not set, the default value is the higher of: - * increase no more than 4 pods per 60 seconds - * double the number of pods per 60 seconds - No stabilization is used. - properties: - policies: - description: |- - policies is a list of potential scaling polices which can be used during scaling. - At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid - items: - description: HPAScalingPolicy is a single policy - which must hold true for a specified past interval. - properties: - periodSeconds: - description: |- - periodSeconds specifies the window of time for which the policy should hold true. - PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). - format: int32 - type: integer - type: - description: type is used to specify the scaling - policy. - type: string - value: - description: |- - value contains the amount of change which is permitted by the policy. - It must be greater than zero - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - x-kubernetes-list-type: atomic - selectPolicy: - description: |- - selectPolicy is used to specify which policy should be used. - If not set, the default value Max is used. - type: string - stabilizationWindowSeconds: - description: |- - stabilizationWindowSeconds is the number of seconds for which past recommendations should be - considered while scaling up or scaling down. - StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). - If not set, use the default values: - - For scale up: 0 (i.e. no stabilization is done). - - For scale down: 300 (i.e. the stabilization window is 300 seconds long). - format: int32 - type: integer - type: object - type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: - description: |- - MetricSpec specifies how to scale based on a single metric - (only `type` and one other matching field should be set at once). - properties: - containerResource: - description: |- - containerResource refers to a resource metric (such as those specified in - requests and limits) known to Kubernetes describing a single container in - each pod of the current scale target (e.g. CPU or memory). Such metrics are - built in to Kubernetes, and have special scaling options on top of those - available to normal per-pod metrics using the "pods" source. - This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. - properties: - container: - description: container is the name of the container - in the pods of the scaling target - type: string - name: - description: name is the name of the resource in - question. - type: string - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - container - - name - - target - type: object - external: - description: |- - external refers to a global metric that is not associated - with any Kubernetes object. It allows autoscaling based on information - coming from components running outside of cluster - (for example length of queue in cloud messaging service, or - QPS from loadbalancer running outside of cluster). - properties: - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - metric - - target - type: object - object: - description: |- - object refers to a metric describing a single kubernetes object - (for example, hits-per-second on an Ingress object). - properties: - describedObject: - description: describedObject specifies the descriptions - of a object,such as kind,name apiVersion - properties: - apiVersion: - description: apiVersion is the API version of - the referent - type: string - kind: - description: 'kind is the kind of the referent; - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'name is the name of the referent; - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - required: - - kind - - name - type: object - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - describedObject - - metric - - target - type: object - pods: - description: |- - pods refers to a metric describing each pod in the current scale target - (for example, transactions-processed-per-second). The values will be - averaged together before being compared to the target value. - properties: - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - metric - - target - type: object - resource: - description: |- - resource refers to a resource metric (such as those specified in - requests and limits) known to Kubernetes describing each pod in the - current scale target (e.g. CPU or memory). Such metrics are built in to - Kubernetes, and have special scaling options on top of those available - to normal per-pod metrics using the "pods" source. - properties: - name: - description: name is the name of the resource in - question. - type: string - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - name - - target - type: object - type: - description: |- - type is the type of metric source. It should be one of "ContainerResource", "External", - "Object", "Pods" or "Resource", each mapping to a matching field in the object. - Note: "ContainerResource" type is available on when the feature-gate - HPAContainerMetrics is enabled - type: string - required: - - type - type: object - type: array - minReplicas: - format: int32 - type: integer - required: - - maxReplicas - type: object - type: object - startupProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - userID: - format: int64 - type: integer - required: - - authSecret - - dataStoreParams - type: object - status: - description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 - properties: - availableReplicas: - format: int32 - type: integer - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - state: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 1a736c2c..4a01d8f8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -63,7 +63,6 @@ rules: - apps.nvidia.com resources: - nemodatastores - - nemodatastorev2s - nemoentitystores - nemoevaluators - nemoguardrails @@ -82,7 +81,6 @@ rules: - apps.nvidia.com resources: - nemodatastores/finalizers - - nemodatastorev2s/finalizers - nemoentitystores/finalizers - nemoevaluators/finalizers - nemoguardrails/finalizers @@ -95,7 +93,6 @@ rules: - apps.nvidia.com resources: - nemodatastores/status - - nemodatastorev2s/status - nemoentitystores/status - nemoevaluators/status - nemoguardrails/status diff --git a/config/samples/apps_v1alpha1_nimdatastorev2.yaml b/config/samples/apps_v1alpha1_nimdatastore.yaml similarity index 91% rename from config/samples/apps_v1alpha1_nimdatastorev2.yaml rename to config/samples/apps_v1alpha1_nimdatastore.yaml index 6a6555f5..c39ef799 100644 --- a/config/samples/apps_v1alpha1_nimdatastorev2.yaml +++ b/config/samples/apps_v1alpha1_nimdatastore.yaml @@ -1,5 +1,5 @@ apiVersion: apps.nvidia.com/v1alpha1 -kind: NemoDatastoreV2 +kind: NemoDatastore metadata: name: nemodatastore spec: @@ -14,13 +14,11 @@ spec: lfsJwtSecret: "nemo-ms-nemo-datastore--lfs-jwt" objStoreSecret: "nds-minio-existing-secret" sshEnabled: false - persistence: - claimName: "pvc-shared-data" + pvc: + name: "pvc-shared-data" create: true - enabled: true storageClass: "local-path" - accessModes: - - ReadWriteOnce + volumeAccessMode: ReadWriteOnce size: "10Gi" expose: service: diff --git a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml index ffdc4923..ae998d7d 100644 --- a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml +++ b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml @@ -65,40 +65,60 @@ spec: type: array dataStoreParams: properties: - appVersion: + datastoreConfigSecret: type: string - databaseHost: + datastoreInitSecret: type: string - databasePort: + datastoreInlineConfigSecret: type: string - databaseURL: + datastoreSettingsSecret: type: string dbSecret: type: string - envConfigmap: + giteaAdminSecret: type: string - envSecret: + lfsJwtSecret: type: string - giteaEndpoint: - type: string - giteaSecret: - type: string - initContainerCommand: - items: - type: string - type: array - initContainerImage: + objStoreSecret: type: string + persistence: + properties: + accessModes: + items: + type: string + type: array + claimName: + type: string + create: + type: boolean + enabled: + type: boolean + labels: + additionalProperties: + type: string + type: object + size: + type: string + storageClass: + type: string + required: + - claimName + - create + - enabled + - size + type: object + sshEnabled: + type: boolean required: - - appVersion - - databaseHost - - databasePort - - databaseURL + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret - dbSecret - - envConfigmap - - envSecret - - giteaEndpoint - - giteaSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - sshEnabled type: object env: items: diff --git a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml deleted file mode 100644 index 518d4f75..00000000 --- a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml +++ /dev/null @@ -1,2230 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.2 - name: nemodatastorev2s.apps.nvidia.com -spec: - group: apps.nvidia.com - names: - kind: NemoDatastoreV2 - listKind: NemoDatastoreV2List - plural: nemodatastorev2s - singular: nemodatastorev2 - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.state - name: Status - type: string - - format: date-time - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 - properties: - annotations: - additionalProperties: - type: string - type: object - args: - items: - type: string - type: array - authSecret: - description: The name of an secret that contains authn for the NGC - NIM service API - type: string - command: - items: - type: string - type: array - dataStoreParams: - properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: - properties: - accessModes: - items: - type: string - type: array - claimName: - type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: - type: string - storageClass: - type: string - required: - - claimName - - create - - enabled - - size - type: object - sshEnabled: - type: boolean - required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled - type: object - env: - items: - description: EnvVar represents an environment variable present in - a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot - be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is - written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified - API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed - resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - expose: - description: Expose defines attributes to expose the service - properties: - ingress: - description: Ingress defines attributes to enable ingress for - the service - properties: - annotations: - additionalProperties: - type: string - type: object - enabled: - description: ingress, or virtualService - not both - type: boolean - spec: - description: IngressSpec describes the Ingress the user wishes - to exist. - properties: - defaultBackend: - description: |- - defaultBackend is the backend that should handle requests that don't - match any rule. If Rules are not specified, DefaultBackend must be specified. - If DefaultBackend is not set, the handling of requests that do not match any - of the rules will be up to the Ingress controller. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - ingressClassName: - description: |- - ingressClassName is the name of an IngressClass cluster resource. Ingress - controller implementations use this field to know whether they should be - serving this Ingress resource, by a transitive connection - (controller -> IngressClass -> Ingress resource). Although the - `kubernetes.io/ingress.class` annotation (simple constant name) was never - formally defined, it was widely supported by Ingress controllers to create - a direct binding between Ingress controller and Ingress resources. Newly - created Ingress resources should prefer using the field. However, even - though the annotation is officially deprecated, for backwards compatibility - reasons, ingress controllers should still honor that annotation if present. - type: string - rules: - description: |- - rules is a list of host rules used to configure the Ingress. If unspecified, - or no rule matches, all traffic is sent to the default backend. - items: - description: |- - IngressRule represents the rules mapping the paths under a specified host to - the related backend services. Incoming requests are first evaluated for a host - match, then routed to the backend associated with the matching IngressRuleValue. - properties: - host: - description: "host is the fully qualified domain - name of a network host, as defined by RFC 3986.\nNote - the following deviations from the \"host\" part - of the\nURI as defined in RFC 3986:\n1. IPs are - not allowed. Currently an IngressRuleValue can - only apply to\n the IP in the Spec of the parent - Ingress.\n2. The `:` delimiter is not respected - because ports are not allowed.\n\t Currently - the port of an Ingress is implicitly :80 for http - and\n\t :443 for https.\nBoth these may change - in the future.\nIncoming requests are matched - against the host before the\nIngressRuleValue. - If the host is unspecified, the Ingress routes - all\ntraffic based on the specified IngressRuleValue.\n\nhost - can be \"precise\" which is a domain name without - the terminating dot of\na network host (e.g. \"foo.bar.com\") - or \"wildcard\", which is a domain name\nprefixed - with a single wildcard label (e.g. \"*.foo.com\").\nThe - wildcard character '*' must appear by itself as - the first DNS label and\nmatches only a single - label. You cannot have a wildcard label by itself - (e.g. Host == \"*\").\nRequests will be matched - against the Host field in the following way:\n1. - If host is precise, the request matches this rule - if the http host header is equal to Host.\n2. - If host is a wildcard, then the request matches - this rule if the http host header\nis to equal - to the suffix (removing the first label) of the - wildcard rule." - type: string - http: - description: |- - HTTPIngressRuleValue is a list of http selectors pointing to backends. - In the example: http:///? -> backend where - where parts of the url correspond to RFC 3986, this resource will be used - to match against everything after the last '/' and before the first '?' - or '#'. - properties: - paths: - description: paths is a collection of paths - that map requests to backends. - items: - description: |- - HTTPIngressPath associates a path with a backend. Incoming urls matching the - path are forwarded to the backend. - properties: - backend: - description: |- - backend defines the referenced service endpoint to which the traffic - will be forwarded to. - properties: - resource: - description: |- - resource is an ObjectRef to another Kubernetes resource in the namespace - of the Ingress object. If resource is specified, a service.Name and - service.Port must not be specified. - This is a mutually exclusive setting with "Service". - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - service: - description: |- - service references a service as a backend. - This is a mutually exclusive setting with "Resource". - properties: - name: - description: |- - name is the referenced service. The service must exist in - the same namespace as the Ingress object. - type: string - port: - description: |- - port of the referenced service. A port name or port number - is required for a IngressServiceBackend. - properties: - name: - description: |- - name is the name of the port on the Service. - This is a mutually exclusive setting with "Number". - type: string - number: - description: |- - number is the numerical port number (e.g. 80) on the Service. - This is a mutually exclusive setting with "Name". - format: int32 - type: integer - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - type: object - path: - description: |- - path is matched against the path of an incoming request. Currently it can - contain characters disallowed from the conventional "path" part of a URL - as defined by RFC 3986. Paths must begin with a '/' and must be present - when using PathType with value "Exact" or "Prefix". - type: string - pathType: - description: |- - pathType determines the interpretation of the path matching. PathType can - be one of the following values: - * Exact: Matches the URL path exactly. - * Prefix: Matches based on a URL path prefix split by '/'. Matching is - done on a path element by element basis. A path element refers is the - list of labels in the path split by the '/' separator. A request is a - match for path p if every p is an element-wise prefix of p of the - request path. Note that if the last element of the path is a substring - of the last element in request path, it is not a match (e.g. /foo/bar - matches /foo/bar/baz, but does not match /foo/barbaz). - * ImplementationSpecific: Interpretation of the Path matching is up to - the IngressClass. Implementations can treat this as a separate PathType - or treat it identically to Prefix or Exact path types. - Implementations are required to support all path types. - type: string - required: - - backend - - pathType - type: object - type: array - x-kubernetes-list-type: atomic - required: - - paths - type: object - type: object - type: array - x-kubernetes-list-type: atomic - tls: - description: |- - tls represents the TLS configuration. Currently the Ingress only supports a - single TLS port, 443. If multiple members of this list specify different hosts, - they will be multiplexed on the same port according to the hostname specified - through the SNI TLS extension, if the ingress controller fulfilling the - ingress supports SNI. - items: - description: IngressTLS describes the transport layer - security associated with an ingress. - properties: - hosts: - description: |- - hosts is a list of hosts included in the TLS certificate. The values in - this list must match the name/s used in the tlsSecret. Defaults to the - wildcard host setting for the loadbalancer controller fulfilling this - Ingress, if left unspecified. - items: - type: string - type: array - x-kubernetes-list-type: atomic - secretName: - description: |- - secretName is the name of the secret used to terminate TLS traffic on - port 443. Field is left optional to allow TLS routing based on SNI - hostname alone. If the SNI host in a listener conflicts with the "Host" - header field used by an IngressRule, the SNI host is used for termination - and value of the "Host" header is used for routing. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - service: - description: Service defines attributes to create a service - properties: - annotations: - additionalProperties: - type: string - type: object - name: - description: override the default service name - type: string - port: - default: 8000 - format: int32 - type: integer - type: - description: Service Type string describes ingress methods - for a service - type: string - required: - - port - type: object - type: object - groupID: - format: int64 - type: integer - image: - description: Image defines image attributes - properties: - pullPolicy: - type: string - pullSecrets: - items: - type: string - type: array - repository: - type: string - tag: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - livenessProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - metrics: - description: Metrics defines attributes to setup metrics collection - properties: - enabled: - type: boolean - serviceMonitor: - description: for use with the Prometheus Operator and the primary - service object - properties: - additionalLabels: - additionalProperties: - type: string - type: object - annotations: - additionalProperties: - type: string - type: object - interval: - description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - scrapeTimeout: - description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` - pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ - type: string - type: object - type: object - nodeSelector: - additionalProperties: - type: string - type: object - podAffinity: - description: Pod affinity is a group of inter pod affinity scheduling - rules. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with - the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - readinessProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - replicas: - default: 1 - minimum: 1 - type: integer - resources: - description: ResourceRequirements describes the compute resource requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - runtimeClass: - type: string - scale: - description: Autoscaling defines attributes to automatically scale - the service based on metrics - properties: - annotations: - additionalProperties: - type: string - type: object - enabled: - type: boolean - hpa: - description: HorizontalPodAutoscalerSpec defines the parameters - required to setup HPA - properties: - behavior: - description: |- - HorizontalPodAutoscalerBehavior configures the scaling behavior of the target - in both Up and Down directions (scaleUp and scaleDown fields respectively). - properties: - scaleDown: - description: |- - scaleDown is scaling policy for scaling Down. - If not set, the default value is to allow to scale down to minReplicas pods, with a - 300 second stabilization window (i.e., the highest recommendation for - the last 300sec is used). - properties: - policies: - description: |- - policies is a list of potential scaling polices which can be used during scaling. - At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid - items: - description: HPAScalingPolicy is a single policy - which must hold true for a specified past interval. - properties: - periodSeconds: - description: |- - periodSeconds specifies the window of time for which the policy should hold true. - PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). - format: int32 - type: integer - type: - description: type is used to specify the scaling - policy. - type: string - value: - description: |- - value contains the amount of change which is permitted by the policy. - It must be greater than zero - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - x-kubernetes-list-type: atomic - selectPolicy: - description: |- - selectPolicy is used to specify which policy should be used. - If not set, the default value Max is used. - type: string - stabilizationWindowSeconds: - description: |- - stabilizationWindowSeconds is the number of seconds for which past recommendations should be - considered while scaling up or scaling down. - StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). - If not set, use the default values: - - For scale up: 0 (i.e. no stabilization is done). - - For scale down: 300 (i.e. the stabilization window is 300 seconds long). - format: int32 - type: integer - type: object - scaleUp: - description: |- - scaleUp is scaling policy for scaling Up. - If not set, the default value is the higher of: - * increase no more than 4 pods per 60 seconds - * double the number of pods per 60 seconds - No stabilization is used. - properties: - policies: - description: |- - policies is a list of potential scaling polices which can be used during scaling. - At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid - items: - description: HPAScalingPolicy is a single policy - which must hold true for a specified past interval. - properties: - periodSeconds: - description: |- - periodSeconds specifies the window of time for which the policy should hold true. - PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). - format: int32 - type: integer - type: - description: type is used to specify the scaling - policy. - type: string - value: - description: |- - value contains the amount of change which is permitted by the policy. - It must be greater than zero - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - x-kubernetes-list-type: atomic - selectPolicy: - description: |- - selectPolicy is used to specify which policy should be used. - If not set, the default value Max is used. - type: string - stabilizationWindowSeconds: - description: |- - stabilizationWindowSeconds is the number of seconds for which past recommendations should be - considered while scaling up or scaling down. - StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). - If not set, use the default values: - - For scale up: 0 (i.e. no stabilization is done). - - For scale down: 300 (i.e. the stabilization window is 300 seconds long). - format: int32 - type: integer - type: object - type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: - description: |- - MetricSpec specifies how to scale based on a single metric - (only `type` and one other matching field should be set at once). - properties: - containerResource: - description: |- - containerResource refers to a resource metric (such as those specified in - requests and limits) known to Kubernetes describing a single container in - each pod of the current scale target (e.g. CPU or memory). Such metrics are - built in to Kubernetes, and have special scaling options on top of those - available to normal per-pod metrics using the "pods" source. - This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. - properties: - container: - description: container is the name of the container - in the pods of the scaling target - type: string - name: - description: name is the name of the resource in - question. - type: string - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - container - - name - - target - type: object - external: - description: |- - external refers to a global metric that is not associated - with any Kubernetes object. It allows autoscaling based on information - coming from components running outside of cluster - (for example length of queue in cloud messaging service, or - QPS from loadbalancer running outside of cluster). - properties: - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - metric - - target - type: object - object: - description: |- - object refers to a metric describing a single kubernetes object - (for example, hits-per-second on an Ingress object). - properties: - describedObject: - description: describedObject specifies the descriptions - of a object,such as kind,name apiVersion - properties: - apiVersion: - description: apiVersion is the API version of - the referent - type: string - kind: - description: 'kind is the kind of the referent; - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'name is the name of the referent; - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - required: - - kind - - name - type: object - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - describedObject - - metric - - target - type: object - pods: - description: |- - pods refers to a metric describing each pod in the current scale target - (for example, transactions-processed-per-second). The values will be - averaged together before being compared to the target value. - properties: - metric: - description: metric identifies the target metric - by name and selector - properties: - name: - description: name is the name of the given metric - type: string - selector: - description: |- - selector is the string-encoded form of a standard kubernetes label selector for the given metric - When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. - When unset, just the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - name - type: object - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - metric - - target - type: object - resource: - description: |- - resource refers to a resource metric (such as those specified in - requests and limits) known to Kubernetes describing each pod in the - current scale target (e.g. CPU or memory). Such metrics are built in to - Kubernetes, and have special scaling options on top of those available - to normal per-pod metrics using the "pods" source. - properties: - name: - description: name is the name of the resource in - question. - type: string - target: - description: target specifies the target value for - the given metric - properties: - averageUtilization: - description: |- - averageUtilization is the target value of the average of the - resource metric across all relevant pods, represented as a percentage of - the requested value of the resource for the pods. - Currently only valid for Resource metric source type - format: int32 - type: integer - averageValue: - anyOf: - - type: integer - - type: string - description: |- - averageValue is the target value of the average of the - metric across all relevant pods (as a quantity) - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: - description: type represents whether the metric - type is Utilization, Value, or AverageValue - type: string - value: - anyOf: - - type: integer - - type: string - description: value is the target value of the - metric (as a quantity). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - type - type: object - required: - - name - - target - type: object - type: - description: |- - type is the type of metric source. It should be one of "ContainerResource", "External", - "Object", "Pods" or "Resource", each mapping to a matching field in the object. - Note: "ContainerResource" type is available on when the feature-gate - HPAContainerMetrics is enabled - type: string - required: - - type - type: object - type: array - minReplicas: - format: int32 - type: integer - required: - - maxReplicas - type: object - type: object - startupProbe: - description: Probe defines attributes for startup/liveness/readiness - probes - properties: - enabled: - type: boolean - probe: - description: |- - Probe describes a health check to be performed against a container to determine whether it is - alive or ready to receive traffic. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header to - be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - type: object - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - userID: - format: int64 - type: integer - required: - - authSecret - - dataStoreParams - type: object - status: - description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 - properties: - availableReplicas: - format: int32 - type: integer - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - state: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/internal/controller/nemo_datastore_controller.go b/internal/controller/nemo_datastore_controller.go index 74567ea1..d9523dea 100644 --- a/internal/controller/nemo_datastore_controller.go +++ b/internal/controller/nemo_datastore_controller.go @@ -274,6 +274,13 @@ func (r *NemoDatastoreReconciler) reconcileNemoDatastore(ctx context.Context, ne "NemoDatastore %s failed, msg: %s", nemoDatastore.Name, err.Error()) } }() + + err = r.reconcilePVC(ctx, nemoDatastore) + if err != nil { + logger.Error(err, "reconciliation of pvc failed", "pvc", nemoDatastore.GetPVCName()) + return ctrl.Result{}, err + } + // Generate annotation for the current operator-version and apply to all resources // Get generic name for all resources namespacedName := types.NamespacedName{Name: nemoDatastore.GetName(), Namespace: nemoDatastore.GetNamespace()} @@ -375,6 +382,10 @@ func (r *NemoDatastoreReconciler) reconcileNemoDatastore(ctx context.Context, ne if len(envFrom) > 0 { result.Spec.Template.Spec.Containers[0].EnvFrom = envFrom } + fsGroup := int64(1000) + result.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ + FSGroup: &fsGroup, + } return result, nil }, "deployment", conditions.ReasonDeploymentFailed) if err != nil { @@ -407,6 +418,41 @@ func (r *NemoDatastoreReconciler) reconcileNemoDatastore(ctx context.Context, ne return ctrl.Result{}, nil } +func (r *NemoDatastoreReconciler) reconcilePVC(ctx context.Context, nemoDatastore *appsv1alpha1.NemoDatastore) error { + logger := r.GetLogger() + pvcName := nemoDatastore.GetPVCName() + pvcNamespacedName := types.NamespacedName{Name: pvcName, Namespace: nemoDatastore.GetNamespace()} + pvc := &corev1.PersistentVolumeClaim{} + err := r.Get(ctx, pvcNamespacedName, pvc) + if err != nil && client.IgnoreNotFound(err) != nil { + return err + } + + // If PVC does not exist, create a new one if creation flag is enabled + if err != nil { + if nemoDatastore.ShouldCreatePersistentStorage() { + pvc, err = shared.ConstructPVC(*nemoDatastore.Spec.DataStoreParams.PVC, metav1.ObjectMeta{Name: pvcName, Namespace: nemoDatastore.GetNamespace()}) + if err != nil { + logger.Error(err, "Failed to construct pvc", "name", pvcName) + return err + } + if err := controllerutil.SetControllerReference(nemoDatastore, pvc, r.GetScheme()); err != nil { + return err + } + err = r.Create(ctx, pvc) + if err != nil { + logger.Error(err, "Failed to create pvc", "name", pvcName) + return err + } + logger.Info("Created PVC for NeMo Datastore", "pvc", pvc.Name) + } else { + logger.Error(err, "PVC doesn't exist and auto-creation is not enabled", "name", pvcNamespacedName) + return err + } + } + return nil +} + func (r *NemoDatastoreReconciler) renderAndSyncResource(ctx context.Context, NemoDatastore client.Object, renderer *render.Renderer, obj client.Object, renderFunc func() (client.Object, error), conditionType string, reason string) error { logger := log.FromContext(ctx) diff --git a/internal/controller/nemo_datastorev2_controller.go b/internal/controller/nemo_datastorev2_controller.go deleted file mode 100644 index f0020902..00000000 --- a/internal/controller/nemo_datastorev2_controller.go +++ /dev/null @@ -1,631 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - "reflect" - - appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" - "github.com/NVIDIA/k8s-nim-operator/internal/conditions" - "github.com/NVIDIA/k8s-nim-operator/internal/k8sutil" - "github.com/NVIDIA/k8s-nim-operator/internal/render" - "github.com/NVIDIA/k8s-nim-operator/internal/shared" - "github.com/NVIDIA/k8s-nim-operator/internal/utils" - "github.com/go-logr/logr" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -// NemoDatastoreV2Finalizer is the finalizer annotation -const NemoDatastoreV2Finalizer = "finalizer.NemoDatastoreV2.apps.nvidia.com" - -// NemoDatastoreV2Reconciler reconciles a NemoDatastoreV2 object -type NemoDatastoreV2Reconciler struct { - client.Client - scheme *runtime.Scheme - log logr.Logger - updater conditions.Updater - renderer render.Renderer - Config *rest.Config - recorder record.EventRecorder - orchestratorType k8sutil.OrchestratorType -} - -// Ensure NemoDatastoreV2Reconciler implements the Reconciler interface -var _ shared.Reconciler = &NemoDatastoreV2Reconciler{} - -// NewNemoDatastoreV2Reconciler creates a new reconciler for NemoDatastoreV2 with the given platform -func NewNemoDatastoreV2Reconciler(client client.Client, scheme *runtime.Scheme, updater conditions.Updater, renderer render.Renderer, log logr.Logger) *NemoDatastoreV2Reconciler { - return &NemoDatastoreV2Reconciler{ - Client: client, - scheme: scheme, - updater: updater, - renderer: renderer, - log: log, - } -} - -// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s/finalizers,verbs=update -// +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions;proxies,verbs=get;list;watch -// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=use,resourceNames=nonroot -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=serviceaccounts;pods;pods/eviction;services;services/finalizers;endpoints,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims;configmaps;secrets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=get;list;watch;create -// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the NemoDatastoreV2 object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.2/pkg/reconcile -func (r *NemoDatastoreV2Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) - defer r.refreshMetrics(ctx) - - // Fetch the NemoDatastoreV2 instance - NemoDatastoreV2 := &appsv1alpha1.NemoDatastoreV2{} - if err := r.Get(ctx, req.NamespacedName, NemoDatastoreV2); err != nil { - if client.IgnoreNotFound(err) != nil { - logger.Error(err, "unable to fetch NemoDatastoreV2", "name", req.NamespacedName) - } - return ctrl.Result{}, client.IgnoreNotFound(err) - } - - logger.Info("Reconciling", "NemoDatastoreV2", NemoDatastoreV2.Name) - - // Check if the instance is marked for deletion - if NemoDatastoreV2.ObjectMeta.DeletionTimestamp.IsZero() { - // Add finalizer if not present - if !controllerutil.ContainsFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) { - controllerutil.AddFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) - if err := r.Update(ctx, NemoDatastoreV2); err != nil { - return ctrl.Result{}, err - } - } - } else { - // The instance is being deleted - if controllerutil.ContainsFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) { - // Perform platform specific cleanup of resources - if err := r.cleanupNemoDatastoreV2(ctx, NemoDatastoreV2); err != nil { - r.GetEventRecorder().Eventf(NemoDatastoreV2, corev1.EventTypeNormal, "Delete", - "NemoDatastoreV2 %s in deleted", NemoDatastoreV2.Name) - return ctrl.Result{}, err - } - - // Remove finalizer to allow for deletion - controllerutil.RemoveFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) - if err := r.Update(ctx, NemoDatastoreV2); err != nil { - r.GetEventRecorder().Eventf(NemoDatastoreV2, corev1.EventTypeNormal, "Delete", - "NemoDatastoreV2 %s finalizer removed", NemoDatastoreV2.Name) - return ctrl.Result{}, err - } - return ctrl.Result{}, nil - } - } - - // Fetch container orchestrator type - _, err := r.GetOrchestratorType() - if err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to get container orchestrator type, %v", err) - } - - // Handle platform-specific reconciliation - if result, err := r.reconcileNemoDatastoreV2(ctx, NemoDatastoreV2); err != nil { - logger.Error(err, "error reconciling NemoDatastoreV2", "name", NemoDatastoreV2.Name) - return result, err - } - - return ctrl.Result{}, nil -} - -func (r *NemoDatastoreV2Reconciler) cleanupNemoDatastoreV2(ctx context.Context, nimService *appsv1alpha1.NemoDatastoreV2) error { - // All dependent (owned) objects will be automatically garbage collected. - // TODO: Handle any custom cleanup logic for the NIM microservice - return nil -} - -// GetScheme returns the scheme of the reconciler -func (r *NemoDatastoreV2Reconciler) GetScheme() *runtime.Scheme { - return r.scheme -} - -// GetLogger returns the logger of the reconciler -func (r *NemoDatastoreV2Reconciler) GetLogger() logr.Logger { - return r.log -} - -// GetClient returns the client instance -func (r *NemoDatastoreV2Reconciler) GetClient() client.Client { - return r.Client -} - -// GetConfig returns the rest config -func (r *NemoDatastoreV2Reconciler) GetConfig() *rest.Config { - return r.Config -} - -// GetUpdater returns the conditions updater instance -func (r *NemoDatastoreV2Reconciler) GetUpdater() conditions.Updater { - return r.updater -} - -// GetRenderer returns the renderer instance -func (r *NemoDatastoreV2Reconciler) GetRenderer() render.Renderer { - return r.renderer -} - -// GetEventRecorder returns the event recorder -func (r *NemoDatastoreV2Reconciler) GetEventRecorder() record.EventRecorder { - return r.recorder -} - -// GetOrchestratorType returns the container platform type -func (r *NemoDatastoreV2Reconciler) GetOrchestratorType() (k8sutil.OrchestratorType, error) { - if r.orchestratorType == "" { - orchestratorType, err := k8sutil.GetOrchestratorType(r.GetClient()) - if err != nil { - return k8sutil.Unknown, fmt.Errorf("Unable to get container orchestrator type, %v", err) - } - r.orchestratorType = orchestratorType - r.GetLogger().Info("Container orchestrator is successfully set", "type", orchestratorType) - } - return r.orchestratorType, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *NemoDatastoreV2Reconciler) SetupWithManager(mgr ctrl.Manager) error { - r.recorder = mgr.GetEventRecorderFor("nemo-datastore-service-controller") - return ctrl.NewControllerManagedBy(mgr). - For(&appsv1alpha1.NemoDatastoreV2{}). - Owns(&appsv1.Deployment{}). - Owns(&appsv1.StatefulSet{}). - Owns(&corev1.Service{}). - Owns(&corev1.ServiceAccount{}). - Owns(&rbacv1.Role{}). - Owns(&rbacv1.RoleBinding{}). - Owns(&networkingv1.Ingress{}). - Owns(&autoscalingv2.HorizontalPodAutoscaler{}). - WithEventFilter(predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - // Type assert to NemoDatastoreV2 - if oldNemoDatastoreV2, ok := e.ObjectOld.(*appsv1alpha1.NemoDatastoreV2); ok { - newNemoDatastoreV2 := e.ObjectNew.(*appsv1alpha1.NemoDatastoreV2) - - // Handle case where object is marked for deletion - if !newNemoDatastoreV2.ObjectMeta.DeletionTimestamp.IsZero() { - return true - } - - // Handle only spec updates - return !reflect.DeepEqual(oldNemoDatastoreV2.Spec, newNemoDatastoreV2.Spec) - } - // For other types we watch, reconcile them - return true - }, - }). - Complete(r) -} - -func (r *NemoDatastoreV2Reconciler) refreshMetrics(ctx context.Context) { - logger := log.FromContext(ctx) - // List all nodes - NemoDatastoreV2List := &appsv1alpha1.NemoDatastoreV2List{} - err := r.Client.List(ctx, NemoDatastoreV2List, &client.ListOptions{}) - if err != nil { - logger.Error(err, "unable to list NemoDatastoreV2s in the cluster") - return - } - //refreshNemoDatastoreV2Metrics(NemoDatastoreV2List) -} - -func (r *NemoDatastoreV2Reconciler) reconcileNemoDatastoreV2(ctx context.Context, nemoDatastore *appsv1alpha1.NemoDatastoreV2) (ctrl.Result, error) { - logger := log.FromContext(ctx) - var err error - defer func() { - if err != nil { - r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeWarning, conditions.Failed, - "NemoDatastoreV2 %s failed, msg: %s", nemoDatastore.Name, err.Error()) - } - }() - - if nemoDatastore.ShouldCreatePersistentStorage() { - pvc, err := nemoDatastore.GetPVC() - if err != nil { - return ctrl.Result{}, err - } - err = r.Create(ctx, pvc) - if err != nil && !errors.IsAlreadyExists(err) { - logger.Error(err, "Failed to create pvc", "name", pvc.Name) - } - } - - // Generate annotation for the current operator-version and apply to all resources - // Get generic name for all resources - namespacedName := types.NamespacedName{Name: nemoDatastore.GetName(), Namespace: nemoDatastore.GetNamespace()} - - renderer := r.GetRenderer() - - // Sync serviceaccount - err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &corev1.ServiceAccount{}, func() (client.Object, error) { - return renderer.ServiceAccount(nemoDatastore.GetServiceAccountParams()) - }, "serviceaccount", conditions.ReasonServiceAccountFailed) - if err != nil { - return ctrl.Result{}, err - } - - // Sync role - err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &rbacv1.Role{}, func() (client.Object, error) { - return renderer.Role(nemoDatastore.GetRoleParams()) - }, "role", conditions.ReasonRoleFailed) - if err != nil { - return ctrl.Result{}, err - } - - // Sync rolebinding - err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &rbacv1.RoleBinding{}, func() (client.Object, error) { - return renderer.RoleBinding(nemoDatastore.GetRoleBindingParams()) - }, "rolebinding", conditions.ReasonRoleBindingFailed) - if err != nil { - return ctrl.Result{}, err - } - - // Sync service - err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &corev1.Service{}, func() (client.Object, error) { - return renderer.Service(nemoDatastore.GetServiceParams()) - }, "service", conditions.ReasonServiceFailed) - if err != nil { - return ctrl.Result{}, err - } - - // Sync ingress - if nemoDatastore.IsIngressEnabled() { - err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &networkingv1.Ingress{}, func() (client.Object, error) { - return renderer.Ingress(nemoDatastore.GetIngressParams()) - }, "ingress", conditions.ReasonIngressFailed) - if err != nil { - return ctrl.Result{}, err - } - } else { - err = r.cleanupResource(ctx, &networkingv1.Ingress{}, namespacedName) - if err != nil && !errors.IsNotFound(err) { - return ctrl.Result{}, err - } - } - - // Sync HPA - if nemoDatastore.IsAutoScalingEnabled() { - err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &autoscalingv2.HorizontalPodAutoscaler{}, func() (client.Object, error) { - return renderer.HPA(nemoDatastore.GetHPAParams()) - }, "hpa", conditions.ReasonHPAFailed) - if err != nil { - return ctrl.Result{}, err - } - } else { - // If autoscaling is disabled, ensure the HPA is deleted - err = r.cleanupResource(ctx, &autoscalingv2.HorizontalPodAutoscaler{}, namespacedName) - if err != nil { - return ctrl.Result{}, err - } - } - - // Sync Service Monitor - if nemoDatastore.IsServiceMonitorEnabled() { - err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &monitoringv1.ServiceMonitor{}, func() (client.Object, error) { - return renderer.ServiceMonitor(nemoDatastore.GetServiceMonitorParams()) - }, "servicemonitor", conditions.ReasonServiceMonitorFailed) - if err != nil { - return ctrl.Result{}, err - } - } - - deploymentParams := nemoDatastore.GetDeploymentParams() - - // Setup volume mounts with model store - deploymentParams.Volumes = nemoDatastore.GetVolumes() - deploymentParams.VolumeMounts = nemoDatastore.GetVolumeMounts() - - logger.Info("Reconciling", "volumes", nemoDatastore.GetVolumes()) - - // Sync deployment - err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &appsv1.Deployment{}, func() (client.Object, error) { - result, err := renderer.Deployment(deploymentParams) - if err != nil { - return nil, err - } - initContainers := nemoDatastore.GetInitContainers() - if len(initContainers) > 0 { - result.Spec.Template.Spec.InitContainers = initContainers - } - envFrom := nemoDatastore.GetEnvFrom() - if len(envFrom) > 0 { - result.Spec.Template.Spec.Containers[0].EnvFrom = envFrom - } - fsGroup := int64(1000) - result.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ - FSGroup: &fsGroup, - } - return result, nil - }, "deployment", conditions.ReasonDeploymentFailed) - if err != nil { - return ctrl.Result{}, err - } - - // Wait for deployment - msg, ready, err := r.isDeploymentReady(ctx, &namespacedName) - if err != nil { - return ctrl.Result{}, err - } - - if !ready { - // Update status as NotReady - err = r.SetConditionsNotReady(ctx, nemoDatastore, conditions.NotReady, msg) - r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeNormal, conditions.NotReady, - "NemoDatastoreV2 %s not ready yet, msg: %s", nemoDatastore.Name, msg) - } else { - // Update status as ready - err = r.SetConditionsReady(ctx, nemoDatastore, conditions.Ready, msg) - r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeNormal, conditions.Ready, - "NemoDatastoreV2 %s ready, msg: %s", nemoDatastore.Name, msg) - } - - if err != nil { - logger.Error(err, "Unable to update status") - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *NemoDatastoreV2Reconciler) renderAndSyncResource(ctx context.Context, NemoDatastoreV2 client.Object, renderer *render.Renderer, obj client.Object, renderFunc func() (client.Object, error), conditionType string, reason string) error { - logger := log.FromContext(ctx) - - namespacedName := types.NamespacedName{Name: NemoDatastoreV2.GetName(), Namespace: NemoDatastoreV2.GetNamespace()} - - resource, err := renderFunc() - if err != nil { - logger.Error(err, "failed to render", conditionType, namespacedName) - statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) - if statusError != nil { - logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) - } - return err - } - - // Check if the resource is nil - if resource == nil { - logger.V(2).Info("rendered nil resource") - return nil - } - - metaAccessor, ok := resource.(metav1.Object) - if !ok { - logger.V(2).Info("rendered un-initialized resource") - return nil - } - - if metaAccessor == nil || metaAccessor.GetName() == "" || metaAccessor.GetNamespace() == "" { - logger.V(2).Info("rendered un-initialized resource") - return nil - } - - if err = controllerutil.SetControllerReference(NemoDatastoreV2, resource, r.GetScheme()); err != nil { - logger.Error(err, "failed to set owner", conditionType, namespacedName) - statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) - if statusError != nil { - logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) - } - return err - } - - err = r.syncResource(ctx, obj, resource, namespacedName) - if err != nil { - logger.Error(err, "failed to sync", conditionType, namespacedName) - statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) - if statusError != nil { - logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) - } - return err - } - return nil -} - -// CheckDeploymentReadiness checks if the Deployment is ready -func (r *NemoDatastoreV2Reconciler) isDeploymentReady(ctx context.Context, namespacedName *types.NamespacedName) (string, bool, error) { - deployment := &appsv1.Deployment{} - err := r.Get(ctx, client.ObjectKey{Name: namespacedName.Name, Namespace: namespacedName.Namespace}, deployment) - if err != nil { - if errors.IsNotFound(err) { - return "", false, nil - } - return "", false, err - } - - cond := getDeploymentCondition(deployment.Status, appsv1.DeploymentProgressing) - if cond != nil && cond.Reason == "ProgressDeadlineExceeded" { - return fmt.Sprintf("deployment %q exceeded its progress deadline", deployment.Name), false, nil - } - if deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas { - return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d out of %d new replicas have been updated...\n", deployment.Name, deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas), false, nil - } - if deployment.Status.Replicas > deployment.Status.UpdatedReplicas { - return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d old replicas are pending termination...\n", deployment.Name, deployment.Status.Replicas-deployment.Status.UpdatedReplicas), false, nil - } - if deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas { - return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d of %d updated replicas are available...\n", deployment.Name, deployment.Status.AvailableReplicas, deployment.Status.UpdatedReplicas), false, nil - } - return fmt.Sprintf("deployment %q successfully rolled out\n", deployment.Name), true, nil -} - -func (r *NemoDatastoreV2Reconciler) syncResource(ctx context.Context, obj client.Object, desired client.Object, namespacedName types.NamespacedName) error { - logger := log.FromContext(ctx) - - err := r.Get(ctx, namespacedName, obj) - if err != nil && !errors.IsNotFound(err) { - return err - } - - if !utils.IsSpecChanged(obj, desired) { - logger.V(2).Info("Object spec has not changed, skipping update", "obj", obj) - return nil - } - logger.V(2).Info("Object spec has changed, updating") - - if errors.IsNotFound(err) { - err = r.Create(ctx, desired) - if err != nil { - return err - } - } else { - err = r.Update(ctx, desired) - if err != nil { - return err - } - } - return nil -} - -// cleanupResource deletes the given Kubernetes resource if it exists. -// If the resource does not exist or an error occurs during deletion, the function returns nil or the error. -// -// Parameters: -// ctx (context.Context): The context for the operation. -// obj (client.Object): The Kubernetes resource to delete. -// namespacedName (types.NamespacedName): The namespaced name of the resource. -// -// Returns: -// error: An error if the resource deletion fails, or nil if the resource is not found or deletion is successful. -func (r *NemoDatastoreV2Reconciler) cleanupResource(ctx context.Context, obj client.Object, namespacedName types.NamespacedName) error { - - logger := log.FromContext(ctx) - - err := r.Get(ctx, namespacedName, obj) - if err != nil && !errors.IsNotFound(err) { - return err - } - - if errors.IsNotFound(err) { - return nil - } - - err = r.Delete(ctx, obj) - if err != nil { - return err - } - logger.V(2).Info("NIM Service object changed, deleting ", "obj", obj) - return nil -} - -func (r *NemoDatastoreV2Reconciler) SetConditionsReady(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { - meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ - Type: conditions.Ready, - Status: metav1.ConditionTrue, - Reason: reason, - Message: message, - }) - - meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ - Type: conditions.Failed, - Status: metav1.ConditionFalse, - Reason: conditions.Ready, - }) - - cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusReady - return r.updateNemoDatastoreV2Status(ctx, cr) -} - -func (r *NemoDatastoreV2Reconciler) SetConditionsNotReady(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { - meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ - Type: conditions.Ready, - Status: metav1.ConditionFalse, - Reason: reason, - Message: message, - }) - - meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ - Type: conditions.Failed, - Status: metav1.ConditionFalse, - Reason: conditions.Ready, - Message: message, - }) - - cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusNotReady - return r.updateNemoDatastoreV2Status(ctx, cr) -} - -func (r *NemoDatastoreV2Reconciler) SetConditionsFailed(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { - meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ - Type: conditions.Ready, - Status: metav1.ConditionFalse, - Reason: conditions.Failed, - }) - - meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ - Type: conditions.Failed, - Status: metav1.ConditionTrue, - Reason: reason, - Message: message, - }) - cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusFailed - return r.updateNemoDatastoreV2Status(ctx, cr) -} - -func (r *NemoDatastoreV2Reconciler) updateNemoDatastoreV2Status(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2) error { - - obj := &appsv1alpha1.NemoDatastoreV2{} - errGet := r.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.GetNamespace()}, obj) - if errGet != nil { - return errGet - } - obj.Status = cr.Status - if err := r.Status().Update(ctx, obj); err != nil { - return err - } - return nil -} From 3346bbdee11c7d4a52796d4d5a607017429f99ed Mon Sep 17 00:00:00 2001 From: Sheng Lin Date: Wed, 22 Jan 2025 17:39:03 -0500 Subject: [PATCH 4/4] Add objectStore and ExternalDatabase fields to datastore .spec Signed-off-by: Sheng Lin --- api/apps/v1alpha1/nemo_datastore_types.go | 188 ++++++++++------- api/apps/v1alpha1/zz_generated.deepcopy.go | 75 +++++-- .../apps.nvidia.com_nemodatastores.yaml | 157 ++++++++------ .../bases/apps.nvidia.com_nemodatastores.yaml | 192 +++++++++++++----- .../samples/apps_v1alpha1_nimdatastore.yaml | 36 +++- .../crds/apps.nvidia.com_nemodatastores.yaml | 157 ++++++++------ .../controller/nemo_datastore_controller.go | 12 +- 7 files changed, 539 insertions(+), 278 deletions(-) diff --git a/api/apps/v1alpha1/nemo_datastore_types.go b/api/apps/v1alpha1/nemo_datastore_types.go index f80be4c0..c5b011ff 100644 --- a/api/apps/v1alpha1/nemo_datastore_types.go +++ b/api/apps/v1alpha1/nemo_datastore_types.go @@ -20,6 +20,7 @@ import ( "fmt" "maps" "os" + "strconv" rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types" utils "github.com/NVIDIA/k8s-nim-operator/internal/utils" @@ -79,7 +80,22 @@ type NemoDatastoreSpec struct { GroupID *int64 `json:"groupID,omitempty"` RuntimeClass string `json:"runtimeClass,omitempty"` - DataStoreParams NemoDatastoreParams `json:"dataStoreParams"` + // ObjectStore specifies the location and credentials for accessing the external Object Storage + ObjectStoreConfig ObjectStoreConfig `json:"objectStoreConfig"` // e.g. minio + // ExternalDatabase contains external PostgreSQL configuration + DatabaseConfig DatabaseConfig `json:"databaseConfig"` // e.g. postgres + // secrets contains the pre-requisite secrets that must be created before deploying the datastore CR + Secrets Secrets `json:"secrets"` + // PVC defines the PersistentVolumeClaim for the datastore + PVC *PersistentVolumeClaim `json:"pvc,omitempty"` +} + +type Secrets struct { + GiteaAdminSecret string `json:"giteaAdminSecret"` + LfsJwtSecret string `json:"lfsJwtSecret"` + DataStoreInitSecret string `json:"datastoreInitSecret"` + DataStoreConfigSecret string `json:"datastoreConfigSecret"` // config_environment.sh + DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"` } // NemoDatastoreStatus defines the observed state of NemoDatastore @@ -89,21 +105,32 @@ type NemoDatastoreStatus struct { State string `json:"state,omitempty"` } -type NemoDatastoreParams struct { - DBSecret string `json:"dbSecret"` - GiteaAdminSecret string `json:"giteaAdminSecret"` +type ObjectStoreConfig struct { // e.g. Minio, s3 + // ObjectStoreCredentials stores the configuration to retrieve the object store credentials + Credentials ObjectStoreCredentials `json:"credentials"` - ObjectStoreSecret string `json:"objStoreSecret"` - DataStoreSettingsSecret string `json:"datastoreSettingsSecret"` - LfsJwtSecret string `json:"lfsJwtSecret"` + // +kubebuilder:default:=true + ServeDirect bool `json:"serveDirect,omitempty"` - DataStoreInitSecret string `json:"datastoreInitSecret"` - DataStoreConfigSecret string `json:"datastoreConfigSecret"` - DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"` + // endpoint is the fully qualidfied object store endpoint + Endpoint string `json:"endpoint"` + // BucketName is the bucket where LFS files will be stored + BucketName string `json:"bucketName"` + // Region is the region where bucket is hosted + Region string `json:"region"` + // SSL enable ssl for object store transport + SSL bool `json:"ssl"` +} - SshEnabled bool `json:"sshEnabled"` +type ObjectStoreCredentials struct { + // User is the non-root username for a NEMO Service in the object store. + User string `json:"user"` - PVC *PersistentVolumeClaim `json:"pvc,omitempty"` + // SecretName is the name of the secret which has the object credentials for a NEMO service user. + SecretName string `json:"secretName"` + + // PasswordKey is the name of the key in the `CredentialsSecret` secret for the object store credentials. + PasswordKey string `json:"passwordKey"` } // +genclient @@ -134,9 +161,8 @@ type NemoDatastoreList struct { // Prefers pvc.Name if explicitly set by the user in the NemoDatastore instance func (n *NemoDatastore) GetPVCName() string { pvcName := fmt.Sprintf("%s-pvc", n.GetName()) - dsParam := n.Spec.DataStoreParams - if dsParam.PVC != nil && dsParam.PVC.Name != "" { - pvcName = dsParam.PVC.Name + if n.Spec.PVC != nil && n.Spec.PVC.Name != "" { + pvcName = n.Spec.PVC.Name } return pvcName } @@ -197,23 +223,16 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { Value: "/data/gitea/git", }, { - Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreKey", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, - }, - }, - }, + Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", + Value: n.Spec.ObjectStoreConfig.Credentials.User, }, { Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreSecret", + Key: n.Spec.ObjectStoreConfig.Credentials.PasswordKey, LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, + Name: n.Spec.ObjectStoreConfig.Credentials.SecretName, }, }, }, @@ -224,7 +243,7 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { SecretKeyRef: &corev1.SecretKeySelector{ Key: "jwtSecret", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.LfsJwtSecret, + Name: n.Spec.Secrets.LfsJwtSecret, }, }, }, @@ -233,9 +252,9 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { Name: "GITEA__DATABASE__PASSWD", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "postgresPassword", + Key: n.Spec.DatabaseConfig.Credentials.PasswordKey, LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DBSecret, + Name: n.Spec.DatabaseConfig.Credentials.SecretName, }, }, }, @@ -245,6 +264,9 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar { } func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { + objStoreSetting := n.Spec.ObjectStoreConfig + dbSetting := n.Spec.DatabaseConfig + envVars := []corev1.EnvVar{ { Name: "GITEA_APP_INI", @@ -271,23 +293,16 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { Value: "/data/gitea/git", }, { - Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreKey", - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, - }, - }, - }, + Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID", + Value: objStoreSetting.Credentials.User, }, { Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "objectStoreSecret", + Key: objStoreSetting.Credentials.PasswordKey, LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.ObjectStoreSecret, + Name: objStoreSetting.Credentials.SecretName, }, }, }, @@ -298,7 +313,7 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { SecretKeyRef: &corev1.SecretKeySelector{ Key: "jwtSecret", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.LfsJwtSecret, + Name: n.Spec.Secrets.LfsJwtSecret, }, }, }, @@ -307,9 +322,9 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { Name: "GITEA__DATABASE__PASSWD", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - Key: "postgresPassword", + Key: dbSetting.Credentials.PasswordKey, LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DBSecret, + Name: dbSetting.Credentials.SecretName, }, }, }, @@ -320,7 +335,7 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { SecretKeyRef: &corev1.SecretKeySelector{ Key: "GITEA_ADMIN_USERNAME", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.GiteaAdminSecret, + Name: n.Spec.Secrets.GiteaAdminSecret, }, }, }, @@ -331,11 +346,55 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar { SecretKeyRef: &corev1.SecretKeySelector{ Key: "GITEA_ADMIN_PASSWORD", LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.GiteaAdminSecret, + Name: n.Spec.Secrets.GiteaAdminSecret, }, }, }, }, + { + Name: "GITEA__LFS__SERVE_DIRECT", + Value: strconv.FormatBool(objStoreSetting.ServeDirect), + }, + { + Name: "GITEA__LFS__STORAGE_TYPE", + Value: "minio", + }, + { + Name: "GITEA__LFS__MINIO_ENDPOINT", + Value: objStoreSetting.Endpoint, + }, + { + Name: "GITEA__LFS__MINIO_BUCKET", + Value: objStoreSetting.BucketName, + }, + { + Name: "GITEA__LFS__MINIO_LOCATION", + Value: objStoreSetting.Region, + }, + { + Name: "GITEA__LFS__MINIO_LOCATION", + Value: objStoreSetting.Region, + }, + { + Name: "GITEA__LFS__MINIO_USE_SSL", + Value: strconv.FormatBool(objStoreSetting.SSL), + }, + { + Name: "GITEA__DATABASE__SSL_MODE", + Value: "disable", + }, + { + Name: "GITEA__DATABASE__NAME", + Value: dbSetting.DatabaseName, + }, + { + Name: "GITEA__DATABASE__HOST", + Value: fmt.Sprintf("%s:%d", dbSetting.Host, dbSetting.Port), + }, + { + Name: "GITEA__DATABASE__USER", + Value: dbSetting.Credentials.User, + }, } return envVars } @@ -369,7 +428,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { Name: "init", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreInitSecret, + SecretName: n.Spec.Secrets.DataStoreInitSecret, DefaultMode: &initMode, }, }, @@ -378,7 +437,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { Name: "config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreConfigSecret, + SecretName: n.Spec.Secrets.DataStoreConfigSecret, DefaultMode: &initMode, }, }, @@ -387,7 +446,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { Name: "inline-config-sources", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: n.Spec.DataStoreParams.DataStoreInlineConfigSecret, + SecretName: n.Spec.Secrets.DataStoreInlineConfigSecret, DefaultMode: &configMode, }, }, @@ -400,7 +459,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { }, } - if n.Spec.DataStoreParams.PVC != nil { + if n.Spec.PVC != nil { volumes = append(volumes, corev1.Volume{ Name: "data", VolumeSource: corev1.VolumeSource{ @@ -421,25 +480,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume { } func (n *NemoDatastore) ShouldCreatePersistentStorage() bool { - return n.Spec.DataStoreParams.PVC != nil && n.Spec.DataStoreParams.PVC.Create != nil && *n.Spec.DataStoreParams.PVC.Create -} - -// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance -func (n *NemoDatastore) GetEnvFrom() []corev1.EnvFromSource { - return []corev1.EnvFromSource{} -} - -// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance -func (n *NemoDatastore) GetInitAppIniEnvFrom() []corev1.EnvFromSource { - return []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: n.Spec.DataStoreParams.DataStoreSettingsSecret, - }, - }, - }, - } + return n.Spec.PVC != nil && n.Spec.PVC.Create != nil && *n.Spec.PVC.Create } // GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance @@ -633,8 +674,8 @@ func (n *NemoDatastore) GetVolumeMounts() []corev1.VolumeMount { Name: "data", } - if n.Spec.DataStoreParams.PVC != nil { - dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath + if n.Spec.PVC != nil { + dataMount.SubPath = n.Spec.PVC.SubPath } mounts = append(mounts, dataMount) return mounts @@ -664,8 +705,8 @@ func (n *NemoDatastore) GetVolumeMountsInitContainer() []corev1.VolumeMount { Name: "data", } - if n.Spec.DataStoreParams.PVC != nil { - dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath + if n.Spec.PVC != nil { + dataMount.SubPath = n.Spec.PVC.SubPath } mounts = append(mounts, dataMount) return mounts @@ -682,7 +723,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container { }, VolumeMounts: n.GetVolumeMountsInitContainer(), Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), }, { Name: "init-app-ini", @@ -693,7 +733,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container { }, VolumeMounts: n.GetVolumeMountsInitContainer(), Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), }, { Name: "configure-datastore", @@ -707,7 +746,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container { }, VolumeMounts: n.GetVolumeMountsInitContainer(), Env: n.GetInitContainerEnv(), - EnvFrom: n.GetInitAppIniEnvFrom(), SecurityContext: &corev1.SecurityContext{ RunAsUser: n.GetUserID(), }, diff --git a/api/apps/v1alpha1/zz_generated.deepcopy.go b/api/apps/v1alpha1/zz_generated.deepcopy.go index d67b4c0b..440b9039 100644 --- a/api/apps/v1alpha1/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/zz_generated.deepcopy.go @@ -1042,26 +1042,6 @@ func (in *NemoDatastoreList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NemoDatastoreParams) DeepCopyInto(out *NemoDatastoreParams) { - *out = *in - if in.PVC != nil { - in, out := &in.PVC, &out.PVC - *out = new(PersistentVolumeClaim) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreParams. -func (in *NemoDatastoreParams) DeepCopy() *NemoDatastoreParams { - if in == nil { - return nil - } - out := new(NemoDatastoreParams) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NemoDatastoreSpec) DeepCopyInto(out *NemoDatastoreSpec) { *out = *in @@ -1137,7 +1117,14 @@ func (in *NemoDatastoreSpec) DeepCopyInto(out *NemoDatastoreSpec) { *out = new(int64) **out = **in } - in.DataStoreParams.DeepCopyInto(&out.DataStoreParams) + out.ObjectStoreConfig = in.ObjectStoreConfig + in.DatabaseConfig.DeepCopyInto(&out.DatabaseConfig) + out.Secrets = in.Secrets + if in.PVC != nil { + in, out := &in.PVC, &out.PVC + *out = new(PersistentVolumeClaim) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreSpec. @@ -1702,6 +1689,37 @@ func (in *NemoGuardrailStatus) DeepCopy() *NemoGuardrailStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreConfig) DeepCopyInto(out *ObjectStoreConfig) { + *out = *in + out.Credentials = in.Credentials +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreConfig. +func (in *ObjectStoreConfig) DeepCopy() *ObjectStoreConfig { + if in == nil { + return nil + } + out := new(ObjectStoreConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreCredentials) DeepCopyInto(out *ObjectStoreCredentials) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreCredentials. +func (in *ObjectStoreCredentials) DeepCopy() *ObjectStoreCredentials { + if in == nil { + return nil + } + out := new(ObjectStoreCredentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PersistentVolumeClaim) DeepCopyInto(out *PersistentVolumeClaim) { *out = *in @@ -1764,6 +1782,21 @@ func (in *Resources) DeepCopy() *Resources { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Secrets) DeepCopyInto(out *Secrets) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Secrets. +func (in *Secrets) DeepCopy() *Secrets { + if in == nil { + return nil + } + out := new(Secrets) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Service) DeepCopyInto(out *Service) { *out = *in diff --git a/bundle/manifests/apps.nvidia.com_nemodatastores.yaml b/bundle/manifests/apps.nvidia.com_nemodatastores.yaml index ae998d7d..4668139e 100644 --- a/bundle/manifests/apps.nvidia.com_nemodatastores.yaml +++ b/bundle/manifests/apps.nvidia.com_nemodatastores.yaml @@ -63,63 +63,6 @@ spec: items: type: string type: array - dataStoreParams: - properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: - properties: - accessModes: - items: - type: string - type: array - claimName: - type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: - type: string - storageClass: - type: string - required: - - claimName - - create - - enabled - - size - type: object - sshEnabled: - type: boolean - required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled - type: object env: items: description: EnvVar represents an environment variable present in @@ -545,6 +488,31 @@ spec: - port type: object type: object + externalDatabase: + properties: + database: + type: string + databaseSecret: + type: string + databaseSecretKey: + type: string + host: + type: string + port: + type: integer + sslMode: + type: string + user: + type: string + required: + - database + - databaseSecret + - databaseSecretKey + - host + - port + - sslMode + - user + type: object groupID: format: int64 type: integer @@ -764,6 +732,34 @@ spec: additionalProperties: type: string type: object + objectStore: + properties: + bucketName: + type: string + endpoint: + type: string + objectStoreSecret: + type: string + objectStoreSecretAccessKey: + type: string + objectStoreSecretAccessSecret: + type: string + region: + type: string + serveDirect: + type: boolean + ssl: + type: boolean + required: + - bucketName + - endpoint + - objectStoreSecret + - objectStoreSecretAccessKey + - objectStoreSecretAccessSecret + - region + - serveDirect + - ssl + type: object podAffinity: description: Pod affinity is a group of inter pod affinity scheduling rules. @@ -1120,6 +1116,30 @@ spec: type: array x-kubernetes-list-type: atomic type: object + pvc: + description: PersistentVolumeClaim defines the attributes of PVC used + as a source for caching NIM model + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during PVC creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. Leave it + as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access mode of the + PVC + type: string + type: object readinessProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -1952,6 +1972,25 @@ spec: - maxReplicas type: object type: object + secrets: + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - giteaAdminSecret + - lfsJwtSecret + type: object startupProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -2156,7 +2195,9 @@ spec: type: integer required: - authSecret - - dataStoreParams + - externalDatabase + - objectStore + - secrets type: object status: description: NemoDatastoreStatus defines the observed state of NemoDatastore diff --git a/config/crd/bases/apps.nvidia.com_nemodatastores.yaml b/config/crd/bases/apps.nvidia.com_nemodatastores.yaml index ae998d7d..d28fc3a0 100644 --- a/config/crd/bases/apps.nvidia.com_nemodatastores.yaml +++ b/config/crd/bases/apps.nvidia.com_nemodatastores.yaml @@ -63,62 +63,62 @@ spec: items: type: string type: array - dataStoreParams: + databaseConfig: + description: ExternalDatabase contains external PostgreSQL configuration properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: + credentials: + description: |- + DatabaseCredentials stores the configuration to retrieve the database credentials. + Required, must not be nil. properties: - accessModes: - items: - type: string - type: array - claimName: + passwordKey: + default: password + description: |- + PasswordKey is the name of the key in the `CredentialsSecret` secret for the database credentials. + Defaults to "password". type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: + secretName: + description: |- + SecretName is the name of the secret which has the database credentials for a NEMO service user. + Required, must not be empty. + minLength: 1 type: string - storageClass: + user: + description: |- + User is the non-root username for a NEMO Service in the database. + Required, must not be empty. + minLength: 1 type: string required: - - claimName - - create - - enabled - - size + - secretName + - user type: object - sshEnabled: - type: boolean + databaseName: + description: |- + DatabaseName is the database name for a NEMO Service. + Required, must not be empty. + minLength: 1 + type: string + host: + description: |- + Host is the hostname of the database. + Required, must not be empty. + minLength: 1 + type: string + port: + default: 5432 + description: |- + Port is the port where the database is reachable at. + If specified, this must be a valid port number, 0 < databasePort < 65536. + Defaults to 5432. + format: int32 + maximum: 65535 + minimum: 1 + type: integer required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled + - credentials + - databaseName + - host type: object env: items: @@ -764,6 +764,54 @@ spec: additionalProperties: type: string type: object + objectStoreConfig: + description: ObjectStore specifies the location and credentials for + accessing the external Object Storage + properties: + bucketName: + description: BucketName is the bucket where LFS files will be + stored + type: string + credentials: + description: ObjectStoreCredentials stores the configuration to + retrieve the object store credentials + properties: + passwordKey: + description: PasswordKey is the name of the key in the `CredentialsSecret` + secret for the object store credentials. + type: string + secretName: + description: SecretName is the name of the secret which has + the object credentials for a NEMO service user. + type: string + user: + description: User is the non-root username for a NEMO Service + in the object store. + type: string + required: + - passwordKey + - secretName + - user + type: object + endpoint: + description: endpoint is the fully qualidfied object store endpoint + type: string + region: + description: Region is the region where bucket is hosted + type: string + serveDirect: + default: true + type: boolean + ssl: + description: SSL enable ssl for object store transport + type: boolean + required: + - bucketName + - credentials + - endpoint + - region + - ssl + type: object podAffinity: description: Pod affinity is a group of inter pod affinity scheduling rules. @@ -1120,6 +1168,29 @@ spec: type: array x-kubernetes-list-type: atomic type: object + pvc: + description: PVC defines the PersistentVolumeClaim for the datastore + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during PVC creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. Leave it + as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access mode of the + PVC + type: string + type: object readinessProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -1952,6 +2023,27 @@ spec: - maxReplicas type: object type: object + secrets: + description: secrets contains the pre-requisite secrets that must + be created before deploying the datastore CR + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - giteaAdminSecret + - lfsJwtSecret + type: object startupProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -2156,7 +2248,9 @@ spec: type: integer required: - authSecret - - dataStoreParams + - databaseConfig + - objectStoreConfig + - secrets type: object status: description: NemoDatastoreStatus defines the observed state of NemoDatastore diff --git a/config/samples/apps_v1alpha1_nimdatastore.yaml b/config/samples/apps_v1alpha1_nimdatastore.yaml index c39ef799..01e2f34c 100644 --- a/config/samples/apps_v1alpha1_nimdatastore.yaml +++ b/config/samples/apps_v1alpha1_nimdatastore.yaml @@ -4,22 +4,36 @@ metadata: name: nemodatastore spec: authSecret: ngc-image-pull-secret - dataStoreParams: + secrets: datastoreConfigSecret: "nemo-ms-nemo-datastore" datastoreInitSecret: "nemo-ms-nemo-datastore-init" datastoreInlineConfigSecret: "nemo-ms-nemo-datastore-inline-config" - datastoreSettingsSecret: "nemo-ms-nemo-datastore-setting" - dbSecret: "nds-pg-existing-secret" giteaAdminSecret: "gitea-admin-credentials" lfsJwtSecret: "nemo-ms-nemo-datastore--lfs-jwt" - objStoreSecret: "nds-minio-existing-secret" - sshEnabled: false - pvc: - name: "pvc-shared-data" - create: true - storageClass: "local-path" - volumeAccessMode: ReadWriteOnce - size: "10Gi" + objectStoreConfig: + credentials: + user: minioUser + secretName: nds-minio-existing-secret + passwordKey: objectStoreSecret + serveDirect: true + endpoint: minio.k8s-nim-operator-system.svc.cluster.local:9000 + bucketName: datastore-dev + region: object-store-region + ssl: false + databaseConfig: + credentials: + user: ndsuser + secretName: nds-pg-existing-secret + passwordKey: postgresPassword + host: nds-pg-postgresql + port: 5432 + databaseName: ndsdb + pvc: + name: "pvc-shared-data" + create: true + storageClass: "local-path" + volumeAccessMode: ReadWriteOnce + size: "10Gi" expose: service: port: 3000 diff --git a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml index ae998d7d..4668139e 100644 --- a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml +++ b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastores.yaml @@ -63,63 +63,6 @@ spec: items: type: string type: array - dataStoreParams: - properties: - datastoreConfigSecret: - type: string - datastoreInitSecret: - type: string - datastoreInlineConfigSecret: - type: string - datastoreSettingsSecret: - type: string - dbSecret: - type: string - giteaAdminSecret: - type: string - lfsJwtSecret: - type: string - objStoreSecret: - type: string - persistence: - properties: - accessModes: - items: - type: string - type: array - claimName: - type: string - create: - type: boolean - enabled: - type: boolean - labels: - additionalProperties: - type: string - type: object - size: - type: string - storageClass: - type: string - required: - - claimName - - create - - enabled - - size - type: object - sshEnabled: - type: boolean - required: - - datastoreConfigSecret - - datastoreInitSecret - - datastoreInlineConfigSecret - - datastoreSettingsSecret - - dbSecret - - giteaAdminSecret - - lfsJwtSecret - - objStoreSecret - - sshEnabled - type: object env: items: description: EnvVar represents an environment variable present in @@ -545,6 +488,31 @@ spec: - port type: object type: object + externalDatabase: + properties: + database: + type: string + databaseSecret: + type: string + databaseSecretKey: + type: string + host: + type: string + port: + type: integer + sslMode: + type: string + user: + type: string + required: + - database + - databaseSecret + - databaseSecretKey + - host + - port + - sslMode + - user + type: object groupID: format: int64 type: integer @@ -764,6 +732,34 @@ spec: additionalProperties: type: string type: object + objectStore: + properties: + bucketName: + type: string + endpoint: + type: string + objectStoreSecret: + type: string + objectStoreSecretAccessKey: + type: string + objectStoreSecretAccessSecret: + type: string + region: + type: string + serveDirect: + type: boolean + ssl: + type: boolean + required: + - bucketName + - endpoint + - objectStoreSecret + - objectStoreSecretAccessKey + - objectStoreSecretAccessSecret + - region + - serveDirect + - ssl + type: object podAffinity: description: Pod affinity is a group of inter pod affinity scheduling rules. @@ -1120,6 +1116,30 @@ spec: type: array x-kubernetes-list-type: atomic type: object + pvc: + description: PersistentVolumeClaim defines the attributes of PVC used + as a source for caching NIM model + properties: + create: + description: Create indicates to create a new PVC + type: boolean + name: + description: Name is the name of the PVC + type: string + size: + description: Size of the NIM cache in Gi, used during PVC creation + type: string + storageClass: + description: StorageClass to be used for PVC creation. Leave it + as empty if the PVC is already created. + type: string + subPath: + type: string + volumeAccessMode: + description: VolumeAccessMode is the volume access mode of the + PVC + type: string + type: object readinessProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -1952,6 +1972,25 @@ spec: - maxReplicas type: object type: object + secrets: + properties: + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + giteaAdminSecret: + type: string + lfsJwtSecret: + type: string + required: + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - giteaAdminSecret + - lfsJwtSecret + type: object startupProbe: description: Probe defines attributes for startup/liveness/readiness probes @@ -2156,7 +2195,9 @@ spec: type: integer required: - authSecret - - dataStoreParams + - externalDatabase + - objectStore + - secrets type: object status: description: NemoDatastoreStatus defines the observed state of NemoDatastore diff --git a/internal/controller/nemo_datastore_controller.go b/internal/controller/nemo_datastore_controller.go index d9523dea..11f79c2b 100644 --- a/internal/controller/nemo_datastore_controller.go +++ b/internal/controller/nemo_datastore_controller.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -378,13 +379,12 @@ func (r *NemoDatastoreReconciler) reconcileNemoDatastore(ctx context.Context, ne if len(initContainers) > 0 { result.Spec.Template.Spec.InitContainers = initContainers } - envFrom := nemoDatastore.GetEnvFrom() - if len(envFrom) > 0 { - result.Spec.Template.Spec.Containers[0].EnvFrom = envFrom + fsGroup := ptr.To[int64](1000) + if nemoDatastore.Spec.GroupID != nil { + fsGroup = nemoDatastore.Spec.GroupID } - fsGroup := int64(1000) result.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ - FSGroup: &fsGroup, + FSGroup: fsGroup, } return result, nil }, "deployment", conditions.ReasonDeploymentFailed) @@ -431,7 +431,7 @@ func (r *NemoDatastoreReconciler) reconcilePVC(ctx context.Context, nemoDatastor // If PVC does not exist, create a new one if creation flag is enabled if err != nil { if nemoDatastore.ShouldCreatePersistentStorage() { - pvc, err = shared.ConstructPVC(*nemoDatastore.Spec.DataStoreParams.PVC, metav1.ObjectMeta{Name: pvcName, Namespace: nemoDatastore.GetNamespace()}) + pvc, err = shared.ConstructPVC(*nemoDatastore.Spec.PVC, metav1.ObjectMeta{Name: pvcName, Namespace: nemoDatastore.GetNamespace()}) if err != nil { logger.Error(err, "Failed to construct pvc", "name", pvcName) return err