diff --git a/api/apps/v1alpha1/nemo_entitystore_types.go b/api/apps/v1alpha1/nemo_entitystore_types.go index 2dfef0b5..ba230c95 100644 --- a/api/apps/v1alpha1/nemo_entitystore_types.go +++ b/api/apps/v1alpha1/nemo_entitystore_types.go @@ -54,12 +54,10 @@ const ( // NemoEntitystoreSpec defines the desired state of NemoEntitystore type NemoEntitystoreSpec struct { - Image Image `json:"image,omitempty"` - Command []string `json:"command,omitempty"` - Args []string `json:"args,omitempty"` - Env []corev1.EnvVar `json:"env,omitempty"` - // The name of an secret that contains authn for the NGC NIM service API - AuthSecret string `json:"authSecret"` + Image Image `json:"image,omitempty"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env []corev1.EnvVar `json:"env,omitempty"` Labels map[string]string `json:"labels,omitempty"` Annotations map[string]string `json:"annotations,omitempty"` NodeSelector map[string]string `json:"nodeSelector,omitempty"` diff --git a/bundle/manifests/apps.nvidia.com_nemoentitystores.yaml b/bundle/manifests/apps.nvidia.com_nemoentitystores.yaml index c5cae793..5d277f04 100644 --- a/bundle/manifests/apps.nvidia.com_nemoentitystores.yaml +++ b/bundle/manifests/apps.nvidia.com_nemoentitystores.yaml @@ -55,10 +55,6 @@ spec: items: type: string type: array - authSecret: - description: The name of an secret that contains authn for the NGC - NIM service API - type: string command: items: type: string @@ -2157,7 +2153,6 @@ spec: format: int64 type: integer required: - - authSecret - databaseConfig type: object status: diff --git a/config/crd/bases/apps.nvidia.com_nemoentitystores.yaml b/config/crd/bases/apps.nvidia.com_nemoentitystores.yaml index c5cae793..5d277f04 100644 --- a/config/crd/bases/apps.nvidia.com_nemoentitystores.yaml +++ b/config/crd/bases/apps.nvidia.com_nemoentitystores.yaml @@ -55,10 +55,6 @@ spec: items: type: string type: array - authSecret: - description: The name of an secret that contains authn for the NGC - NIM service API - type: string command: items: type: string @@ -2157,7 +2153,6 @@ spec: format: int64 type: integer required: - - authSecret - databaseConfig type: object status: diff --git a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemoentitystores.yaml b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemoentitystores.yaml index c5cae793..5d277f04 100644 --- a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemoentitystores.yaml +++ b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemoentitystores.yaml @@ -55,10 +55,6 @@ spec: items: type: string type: array - authSecret: - description: The name of an secret that contains authn for the NGC - NIM service API - type: string command: items: type: string @@ -2157,7 +2153,6 @@ spec: format: int64 type: integer required: - - authSecret - databaseConfig type: object status: diff --git a/internal/controller/nemo_entitystore_controller_test.go b/internal/controller/nemo_entitystore_controller_test.go new file mode 100644 index 00000000..8db22a26 --- /dev/null +++ b/internal/controller/nemo_entitystore_controller_test.go @@ -0,0 +1,330 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" + "github.com/NVIDIA/k8s-nim-operator/internal/conditions" + "github.com/NVIDIA/k8s-nim-operator/internal/render" +) + +var _ = Describe("NemoEntitystore Controller", func() { + var ( + reconciler *NemoEntitystoreReconciler + + client crclient.Client + scheme *runtime.Scheme + updater conditions.Updater + renderer render.Renderer + ) + + // status causes. + var ( + requiredDatabaseConfigCause = metav1.StatusCause{ + Type: "FieldValueRequired", + Message: "Required value", + Field: "spec.databaseConfig", + } + requiredDatabaseHostCause = metav1.StatusCause{ + Type: "FieldValueRequired", + Message: "Required value", + Field: "spec.databaseConfig.host", + } + requiredDatabaseNameCause = metav1.StatusCause{ + Type: "FieldValueRequired", + Message: "Required value", + Field: "spec.databaseConfig.databaseName", + } + invalidDatabasePortCause = metav1.StatusCause{ + Type: "FieldValueInvalid", + Message: "Invalid value: 65536: spec.databaseConfig.port in body should be less than or equal to 65535", + Field: "spec.databaseConfig.port", + } + requiredCredentialsCause = metav1.StatusCause{ + Type: "FieldValueRequired", + Message: "Required value", + Field: "spec.databaseConfig.credentials", + } + requiredUserCause = metav1.StatusCause{ + Type: "FieldValueRequired", + Message: "Required value", + Field: "spec.databaseConfig.credentials.user", + } + requiredSecretNameCause = metav1.StatusCause{ + Type: "FieldValueRequired", + Message: "Required value", + Field: "spec.databaseConfig.credentials.secretName", + } + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(appsv1alpha1.AddToScheme(scheme)).To(Succeed()) + Expect(monitoringv1.AddToScheme(scheme)).To(Succeed()) + Expect(appsv1.AddToScheme(scheme)).To(Succeed()) + Expect(autoscalingv2.AddToScheme(scheme)).To(Succeed()) + Expect(batchv1.AddToScheme(scheme)).To(Succeed()) + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + Expect(networkingv1.AddToScheme(scheme)).To(Succeed()) + Expect(rbacv1.AddToScheme(scheme)).To(Succeed()) + + client = fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&appsv1alpha1.NemoEntitystore{}). + WithStatusSubresource(&appsv1.Deployment{}). + Build() + updater = conditions.NewUpdater(client) + manifestsDir, err := filepath.Abs("../../manifests") + Expect(err).ToNot(HaveOccurred()) + renderer = render.NewRenderer(manifestsDir) + reconciler = &NemoEntitystoreReconciler{ + Client: client, + scheme: scheme, + updater: updater, + renderer: renderer, + recorder: record.NewFakeRecorder(1000), + } + }) + + Context("When reconciling a resource", func() { + const resourceName = "test-resource" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", + } + nemoEntityStore := &appsv1alpha1.NemoEntitystore{} + + BeforeEach(func() { + By("creating the custom resource for the Kind NemoEntitystore") + err := k8sClient.Get(ctx, typeNamespacedName, nemoEntityStore) + Expect(err).To(HaveOccurred()) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + + AfterEach(func() { + resource := &appsv1alpha1.NemoEntitystore{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + if err != nil { + Expect(errors.IsNotFound(err)).To(BeTrue()) + } else { + By("Cleanup the specific resource instance NemoEntitystore") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + + Expect(CleanupChildEntities(ctx, client, typeNamespacedName, schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"})).To(Succeed()) + Expect(CleanupChildEntities(ctx, client, typeNamespacedName, schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"})).To(Succeed()) + Expect(CleanupChildEntities(ctx, client, typeNamespacedName, schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ServiceAccount"})).To(Succeed()) + Expect(CleanupChildEntities(ctx, client, typeNamespacedName, schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "Role"})).To(Succeed()) + Expect(CleanupChildEntities(ctx, client, typeNamespacedName, schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "RoleBinding"})).To(Succeed()) + Expect(CleanupChildEntities(ctx, client, typeNamespacedName, schema.GroupVersionKind{Group: "networking.k8s.io", Version: "v1", Kind: "Ingress"})).To(Succeed()) + Expect(CleanupChildEntities(ctx, client, typeNamespacedName, schema.GroupVersionKind{Group: "autoscaling", Version: "v1", Kind: "HorizontalPodAutoscaler"})).To(Succeed()) + Expect(CleanupChildEntities(ctx, client, typeNamespacedName, schema.GroupVersionKind{Group: "monitoring.coreos.com", Version: "v1", Kind: "ServiceMonitor"})).To(Succeed()) + }) + + It("should reject the resource if databaseConfig is missing", func() { + resource := &appsv1alpha1.NemoEntitystore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: appsv1alpha1.NemoEntitystoreSpec{}, + } + By("creating the custom resource for the Kind NemoEntitystore") + err := k8sClient.Create(ctx, resource) + Expect(err).To(HaveOccurred()) + Expect(errors.IsInvalid(err)).To(BeTrue()) + statusErr := err.(*errors.StatusError) + Expect(statusErr.ErrStatus.Details.Causes).To(ContainElement(requiredDatabaseConfigCause)) + }) + + It("should reject the resource if databaseConfig is missing required fields", func() { + resource := &appsv1alpha1.NemoEntitystore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: appsv1alpha1.NemoEntitystoreSpec{ + DatabaseConfig: &appsv1alpha1.DatabaseConfig{}, + }, + } + By("creating the custom resource for the Kind NemoEntitystore") + err := k8sClient.Create(ctx, resource) + Expect(err).To(HaveOccurred()) + Expect(errors.IsInvalid(err)).To(BeTrue()) + statusErr := err.(*errors.StatusError) + Expect(statusErr.ErrStatus.Details.Causes).To(ContainElements(requiredDatabaseHostCause, requiredDatabaseNameCause, requiredCredentialsCause)) + }) + + It("should reject the resource if databasePort is invalid", func() { + resource := &appsv1alpha1.NemoEntitystore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: appsv1alpha1.NemoEntitystoreSpec{ + DatabaseConfig: &appsv1alpha1.DatabaseConfig{ + Host: "test-pg-host", + DatabaseName: "test-pg-database", + Port: 65536, + Credentials: &appsv1alpha1.DatabaseCredentials{ + User: "test-pg-user", + SecretName: "test-pg-secret", + }, + }, + }, + } + By("creating the custom resource for the Kind NemoEntitystore") + err := k8sClient.Create(ctx, resource) + Expect(err).To(HaveOccurred()) + Expect(errors.IsInvalid(err)).To(BeTrue()) + statusErr := err.(*errors.StatusError) + Expect(statusErr.ErrStatus.Details.Causes).To(HaveLen(1)) + Expect(statusErr.ErrStatus.Details.Causes).To(ContainElement(invalidDatabasePortCause)) + }) + + It("should reject the resource if databaseConfig credentails is missing required fields", func() { + resource := &appsv1alpha1.NemoEntitystore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: appsv1alpha1.NemoEntitystoreSpec{ + DatabaseConfig: &appsv1alpha1.DatabaseConfig{ + Host: "test-pg-host", + DatabaseName: "test-pg-database", + Credentials: &appsv1alpha1.DatabaseCredentials{}, + }, + }, + } + By("creating the custom resource for the Kind NemoEntitystore") + err := k8sClient.Create(ctx, resource) + Expect(err).To(HaveOccurred()) + Expect(errors.IsInvalid(err)).To(BeTrue()) + statusErr := err.(*errors.StatusError) + Expect(statusErr.ErrStatus.Details.Causes).To(ContainElements(requiredSecretNameCause, requiredUserCause)) + }) + + It("should reject the resource if databaseConfig credentails is invalid", func() { + resource := &appsv1alpha1.NemoEntitystore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: appsv1alpha1.NemoEntitystoreSpec{ + DatabaseConfig: &appsv1alpha1.DatabaseConfig{ + Host: "test-pg-host", + DatabaseName: "test-pg-database", + Credentials: &appsv1alpha1.DatabaseCredentials{ + User: "", + SecretName: "", + }, + }, + }, + } + By("creating the custom resource for the Kind NemoEntitystore") + err := k8sClient.Create(ctx, resource) + Expect(err).To(HaveOccurred()) + Expect(errors.IsInvalid(err)).To(BeTrue()) + statusErr := err.(*errors.StatusError) + Expect(statusErr.ErrStatus.Details.Causes).To(ContainElements(requiredSecretNameCause, requiredUserCause)) + }) + + It("should successfully reconcile the NemoEntityStore resource", func() { + resource := &appsv1alpha1.NemoEntitystore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: appsv1alpha1.NemoEntitystoreSpec{ + Image: appsv1alpha1.Image{ + Repository: "test-repo", + Tag: "test-tag", + }, + Replicas: 1, + DatabaseConfig: &appsv1alpha1.DatabaseConfig{ + Host: "test-pg-host", + DatabaseName: "test-pg-database", + Credentials: &appsv1alpha1.DatabaseCredentials{ + User: "test-pg-user", + SecretName: "test-pg-secret", + }, + }, + }, + } + By("creating the custom resource for the Kind NemoEntitystore") + err := client.Create(ctx, resource) + Expect(err).ToNot(HaveOccurred()) + + By("Reconciling the created resource") + _, err = reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + + err = client.Get(ctx, typeNamespacedName, nemoEntityStore) + Expect(err).ToNot(HaveOccurred()) + Expect(nemoEntityStore.Finalizers).To(ContainElement(NemoEntitystoreFinalizer)) + Expect(nemoEntityStore.Status.State).To(Equal(appsv1alpha1.NemoEntitystoreStatusNotReady)) + + // Deployment should exist. + esDeploy := &appsv1.Deployment{} + err = client.Get(ctx, typeNamespacedName, esDeploy) + Expect(err).ToNot(HaveOccurred()) + + By("deleting the custom resource") + err = client.Delete(ctx, resource) + Expect(err).ToNot(HaveOccurred()) + + By("Reconciling the deleted resource") + _, err = reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + + // Nemo Entitystore CR should not exist. + Eventually(func() bool { + nemoEntityStore = &appsv1alpha1.NemoEntitystore{} + err := client.Get(ctx, typeNamespacedName, nemoEntityStore) + return err != nil && errors.IsNotFound(err) + }, time.Second*5, time.Millisecond*500).Should(BeTrue()) + }) + }) +}) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 2b3fdc37..916babc5 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2024. +Copyright 2024-2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "context" "fmt" "path/filepath" "runtime" @@ -25,6 +26,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -88,3 +92,25 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) + +func CleanupChildEntities(ctx context.Context, c client.Client, namespacedName types.NamespacedName, gvk schema.GroupVersionKind) error { + // List all potential child resources + childList := &unstructured.UnstructuredList{} + childList.SetGroupVersionKind(gvk) + if err := c.List(ctx, childList); err != nil { + return err + } + + // Delete children with matching owner references + for _, child := range childList.Items { + for _, owner := range child.GetOwnerReferences() { + if owner.Name == namespacedName.Name { + if err := c.Delete(ctx, &child); err != nil { + return err + } + } + } + } + + return nil +}