From aa32751de248dfa9c71879caa54ee9e3caf248a0 Mon Sep 17 00:00:00 2001 From: kevin1689 Date: Fri, 28 Apr 2023 01:45:19 +0100 Subject: [PATCH] add: Support the deletion protection of service and ingress Signed-off-by: kevin1689 --- config/webhook/manifests.yaml | 41 +++++ config/webhook/patch_manifests.yaml | 10 ++ pkg/features/kruise_features.go | 4 +- pkg/webhook/add_ingress.go | 29 ++++ pkg/webhook/add_service.go | 29 ++++ .../ingress/validating/ingress_handler.go | 87 +++++++++++ pkg/webhook/ingress/validating/webhooks.go | 28 ++++ .../service/validating/service_handler.go | 74 +++++++++ pkg/webhook/service/validating/webhooks.go | 28 ++++ .../deletionprotection/deletion_protection.go | 24 +++ test/e2e/policy/deletionprotection.go | 143 ++++++++++++++++++ 11 files changed, 495 insertions(+), 2 deletions(-) create mode 100644 pkg/webhook/add_ingress.go create mode 100644 pkg/webhook/add_service.go create mode 100644 pkg/webhook/ingress/validating/ingress_handler.go create mode 100644 pkg/webhook/ingress/validating/webhooks.go create mode 100644 pkg/webhook/service/validating/service_handler.go create mode 100644 pkg/webhook/service/validating/webhooks.go diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 7cd14510e7..72b8c435da 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -474,6 +474,27 @@ webhooks: resources: - imagepulljobs sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-ingress + failurePolicy: Fail + name: vingress.kb.io + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + - v1beta1 + operations: + - DELETE + resources: + - ingresses + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -619,6 +640,26 @@ webhooks: resources: - podunavailablebudgets sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-service + failurePolicy: Fail + name: vservice.kb.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - DELETE + resources: + - services + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/config/webhook/patch_manifests.yaml b/config/webhook/patch_manifests.yaml index b2423e404a..02ed68c702 100644 --- a/config/webhook/patch_manifests.yaml +++ b/config/webhook/patch_manifests.yaml @@ -39,6 +39,16 @@ webhooks: matchExpressions: - key: policy.kruise.io/delete-protection operator: Exists +- name: vservice.kb.io + objectSelector: + matchExpressions: + - key: policy.kruise.io/delete-protection + operator: Exists +- name: vingress.kb.io + objectSelector: + matchExpressions: + - key: policy.kruise.io/delete-protection + operator: Exists - name: vpod.kb.io namespaceSelector: matchExpressions: diff --git a/pkg/features/kruise_features.go b/pkg/features/kruise_features.go index 493b7b9b23..712cb2d8c7 100644 --- a/pkg/features/kruise_features.go +++ b/pkg/features/kruise_features.go @@ -52,10 +52,10 @@ const ( CloneSetPartitionRollback featuregate.Feature = "CloneSetPartitionRollback" // ResourcesDeletionProtection enables protection for resources deletion, currently supports - // Namespace, CustomResourcesDefinition, Deployment, StatefulSet, ReplicaSet, CloneSet, Advanced StatefulSet, UnitedDeployment. + // Namespace, Service, Ingress, CustomResourcesDefinition, Deployment, StatefulSet, ReplicaSet, CloneSet, Advanced StatefulSet, UnitedDeployment. // It is only supported for Kubernetes version >= 1.16 // Note that if it is enabled during Kruise installation or upgrade, Kruise will require more authorities: - // 1. Webhook for deletion operation of namespace, crd, deployment, statefulset, replicaset and workloads in Kruise. + // 1. Webhook for deletion operation of namespace, service, ingress, crd, deployment, statefulset, replicaset and workloads in Kruise. // 2. ClusterRole for reading all resource types, because CRD validation needs to list the CRs of this CRD. ResourcesDeletionProtection featuregate.Feature = "ResourcesDeletionProtection" diff --git a/pkg/webhook/add_ingress.go b/pkg/webhook/add_ingress.go new file mode 100644 index 0000000000..c214e24e79 --- /dev/null +++ b/pkg/webhook/add_ingress.go @@ -0,0 +1,29 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openkruise/kruise/pkg/features" + utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "github.com/openkruise/kruise/pkg/webhook/ingress/validating" +) + +func init() { + addHandlersWithGate(validating.HandlerMap, func() (enabled bool) { + return utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) + }) +} diff --git a/pkg/webhook/add_service.go b/pkg/webhook/add_service.go new file mode 100644 index 0000000000..a6de904d35 --- /dev/null +++ b/pkg/webhook/add_service.go @@ -0,0 +1,29 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openkruise/kruise/pkg/features" + utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "github.com/openkruise/kruise/pkg/webhook/service/validating" +) + +func init() { + addHandlersWithGate(validating.HandlerMap, func() (enabled bool) { + return utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) + }) +} diff --git a/pkg/webhook/ingress/validating/ingress_handler.go b/pkg/webhook/ingress/validating/ingress_handler.go new file mode 100644 index 0000000000..6e005eb528 --- /dev/null +++ b/pkg/webhook/ingress/validating/ingress_handler.go @@ -0,0 +1,87 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "context" + "net/http" + + "github.com/openkruise/kruise/pkg/webhook/util/deletionprotection" + admissionv1 "k8s.io/api/admission/v1" + networkingv1 "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type IngressHandler struct { + Client client.Client + + // Decoder decodes objects + Decoder *admission.Decoder +} + +var _ admission.Handler = &IngressHandler{} + +// Handle handles admission requests. +func (h *IngressHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + if req.AdmissionRequest.Operation != admissionv1.Delete || req.AdmissionRequest.SubResource != "" { + return admission.ValidationResponse(true, "") + } + if len(req.OldObject.Raw) == 0 { + klog.Warningf("Skip to validate ingress %s deletion for no old object, maybe because of Kubernetes version < 1.16", req.Name) + return admission.ValidationResponse(true, "") + } + + var metaObj metav1.Object + switch req.Kind.Version { + case "v1beta1": + obj := &networkingv1beta1.Ingress{} + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + metaObj = obj + case "v1": + obj := &networkingv1.Ingress{} + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + metaObj = obj + } + + if err := deletionprotection.ValidateIngressDeletion(metaObj); err != nil { + return admission.Errored(http.StatusForbidden, err) + } + return admission.ValidationResponse(true, "") +} + +var _ inject.Client = &IngressHandler{} + +func (h *IngressHandler) InjectClient(c client.Client) error { + h.Client = c + return nil +} + +var _ admission.DecoderInjector = &IngressHandler{} + +func (h *IngressHandler) InjectDecoder(d *admission.Decoder) error { + h.Decoder = d + return nil +} diff --git a/pkg/webhook/ingress/validating/webhooks.go b/pkg/webhook/ingress/validating/webhooks.go new file mode 100644 index 0000000000..4e5c31d9a9 --- /dev/null +++ b/pkg/webhook/ingress/validating/webhooks.go @@ -0,0 +1,28 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +// +kubebuilder:webhook:path=/validate-ingress,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=networking.k8s.io,resources=ingresses,verbs=delete,versions=v1;v1beta1,name=vingress.kb.io + +var ( + // HandlerMap contains admission webhook handlers + HandlerMap = map[string]admission.Handler{ + "validate-ingress": &IngressHandler{}, + } +) diff --git a/pkg/webhook/service/validating/service_handler.go b/pkg/webhook/service/validating/service_handler.go new file mode 100644 index 0000000000..6b073f4d96 --- /dev/null +++ b/pkg/webhook/service/validating/service_handler.go @@ -0,0 +1,74 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "context" + "net/http" + + "github.com/openkruise/kruise/pkg/webhook/util/deletionprotection" + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type ServiceHandler struct { + Client client.Client + + // Decoder decodes objects + Decoder *admission.Decoder +} + +var _ admission.Handler = &ServiceHandler{} + +// Handle handles admission requests. +func (h *ServiceHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + if req.AdmissionRequest.Operation != admissionv1.Delete || req.AdmissionRequest.SubResource != "" { + return admission.ValidationResponse(true, "") + } + if len(req.OldObject.Raw) == 0 { + klog.Warningf("Skip to validate service %s deletion for no old object, maybe because of Kubernetes version < 1.16", req.Name) + return admission.ValidationResponse(true, "") + } + + obj := &v1.Service{} + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if err := deletionprotection.ValidateServiceDeletion(obj); err != nil { + return admission.Errored(http.StatusForbidden, err) + } + return admission.ValidationResponse(true, "") +} + +var _ inject.Client = &ServiceHandler{} + +func (h *ServiceHandler) InjectClient(c client.Client) error { + h.Client = c + return nil +} + +var _ admission.DecoderInjector = &ServiceHandler{} + +func (h *ServiceHandler) InjectDecoder(d *admission.Decoder) error { + h.Decoder = d + return nil +} diff --git a/pkg/webhook/service/validating/webhooks.go b/pkg/webhook/service/validating/webhooks.go new file mode 100644 index 0000000000..9461ffd773 --- /dev/null +++ b/pkg/webhook/service/validating/webhooks.go @@ -0,0 +1,28 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +// +kubebuilder:webhook:path=/validate-service,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups="",resources=services,verbs=delete,versions=v1,name=vservice.kb.io + +var ( + // HandlerMap contains admission webhook handlers + HandlerMap = map[string]admission.Handler{ + "validate-service": &ServiceHandler{}, + } +) diff --git a/pkg/webhook/util/deletionprotection/deletion_protection.go b/pkg/webhook/util/deletionprotection/deletion_protection.go index 7928643083..d48304e42b 100644 --- a/pkg/webhook/util/deletionprotection/deletion_protection.go +++ b/pkg/webhook/util/deletionprotection/deletion_protection.go @@ -50,6 +50,30 @@ func ValidateWorkloadDeletion(obj metav1.Object, replicas *int32) error { return nil } +func ValidateServiceDeletion(service *v1.Service) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || service.DeletionTimestamp != nil { + return nil + } + switch val := service.Labels[policyv1alpha1.DeletionProtectionKey]; val { + case policyv1alpha1.DeletionProtectionTypeAlways: + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s", policyv1alpha1.DeletionProtectionKey, val) + default: + } + return nil +} + +func ValidateIngressDeletion(obj metav1.Object) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || obj.GetDeletionTimestamp() != nil { + return nil + } + switch val := obj.GetLabels()[policyv1alpha1.DeletionProtectionKey]; val { + case policyv1alpha1.DeletionProtectionTypeAlways: + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s", policyv1alpha1.DeletionProtectionKey, val) + default: + } + return nil +} + func ValidateNamespaceDeletion(c client.Client, namespace *v1.Namespace) error { if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || namespace.DeletionTimestamp != nil { return nil diff --git a/test/e2e/policy/deletionprotection.go b/test/e2e/policy/deletionprotection.go index 0fd46a2a27..1398881287 100644 --- a/test/e2e/policy/deletionprotection.go +++ b/test/e2e/policy/deletionprotection.go @@ -29,6 +29,8 @@ import ( "github.com/openkruise/kruise/test/e2e/framework" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/resource" @@ -36,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -371,4 +374,144 @@ var _ = SIGDescribe("DeletionProtection", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) + + framework.KruiseDescribe("Service deletion", func() { + ginkgo.It("should be protected", func() { + ginkgo.By("Create a Service with Always") + name := "svc-" + randStr + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: map[string]string{policyv1alpha1.DeletionProtectionKey: policyv1alpha1.DeletionProtectionTypeAlways}, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"owner": "foo"}, + Ports: []v1.ServicePort{ + {Port: 80, Name: "http", Protocol: v1.ProtocolTCP}, + }, + }, + } + _, err := c.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Service should be rejected") + err = c.CoreV1().Services(ns).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Patch the Service deletion to null") + _, err = c.CoreV1().Services(ns).Patch(context.TODO(), svc.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":null}}}`, policyv1alpha1.DeletionProtectionKey)), metav1.PatchOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Service successfully") + err = c.CoreV1().Services(ns).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + framework.KruiseDescribe("Ingress deletion", func() { + ginkgo.It("should be protected", func() { + ginkgo.By("Create a Ingress with Always") + name := "ing-" + randStr + pathType := networkingv1.PathTypePrefix + ing := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{policyv1alpha1.DeletionProtectionKey: policyv1alpha1.DeletionProtectionTypeAlways}, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err := c.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + + // for the cluster using old Kubernetes version, use networking.k8s.io/v1beta1 instead of networking.k8s.io/v1 to create Ingress resource + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + pathType := networkingv1beta1.PathTypePrefix + ing := &networkingv1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{policyv1alpha1.DeletionProtectionKey: policyv1alpha1.DeletionProtectionTypeAlways}, + }, + Spec: networkingv1beta1.IngressSpec{ + Rules: []networkingv1beta1.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networkingv1beta1.IngressRuleValue{ + HTTP: &networkingv1beta1.HTTPIngressRuleValue{ + Paths: []networkingv1beta1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: networkingv1beta1.IngressBackend{ + ServiceName: "test", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err = c.NetworkingV1beta1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Ingress should be rejected") + err = c.NetworkingV1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + err = c.NetworkingV1beta1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + } + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Patch the Ingress deletion to null") + _, err = c.NetworkingV1().Ingresses(ns).Patch(context.TODO(), ing.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":null}}}`, policyv1alpha1.DeletionProtectionKey)), metav1.PatchOptions{}) + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + _, err = c.NetworkingV1beta1().Ingresses(ns).Patch(context.TODO(), ing.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":null}}}`, policyv1alpha1.DeletionProtectionKey)), metav1.PatchOptions{}) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Ingress successfully") + err = c.NetworkingV1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + err = c.NetworkingV1beta1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) })