diff --git a/go.mod b/go.mod
index a032095a..4a1e3018 100644
--- a/go.mod
+++ b/go.mod
@@ -67,7 +67,6 @@ require (
 	k8s.io/api v0.28.6
 	k8s.io/apimachinery v0.29.2
 	k8s.io/client-go v12.0.0+incompatible
-	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	kubevirt.io/api v1.1.1
 )
 
@@ -216,6 +215,7 @@ require (
 	k8s.io/kube-aggregator v0.26.4 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/mount-utils v0.30.0 // indirect
+	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
 	kubevirt.io/client-go v1.1.1 // indirect
 	kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect
 	kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
diff --git a/pkg/lb/servicelb/manager.go b/pkg/lb/servicelb/manager.go
index 3f1fff96..4b5773de 100644
--- a/pkg/lb/servicelb/manager.go
+++ b/pkg/lb/servicelb/manager.go
@@ -17,7 +17,6 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/utils/pointer"
 
 	"github.com/harvester/harvester-load-balancer/pkg/apis/loadbalancer.harvesterhci.io"
 	lbv1 "github.com/harvester/harvester-load-balancer/pkg/apis/loadbalancer.harvesterhci.io/v1beta1"
@@ -567,6 +566,7 @@ const dummyEndpointIPv4Address = "10.52.0.255"
 const dummyEndpointID = "dummy347-546a-4642-9da6-5608endpoint"
 
 func appendDummyEndpoint(eps []discoveryv1.Endpoint, lb *lbv1.LoadBalancer) []discoveryv1.Endpoint {
+	cond := true
 	endpoint := discoveryv1.Endpoint{
 		Addresses: []string{dummyEndpointIPv4Address},
 		TargetRef: &corev1.ObjectReference{
@@ -575,7 +575,7 @@ func appendDummyEndpoint(eps []discoveryv1.Endpoint, lb *lbv1.LoadBalancer) []di
 			UID:       dummyEndpointID,
 		},
 		Conditions: discoveryv1.EndpointConditions{
-			Ready: pointer.Bool(true),
+			Ready: &cond,
 		},
 	}
 	eps = append(eps, endpoint)
@@ -642,6 +642,7 @@ func (m *Manager) constructEndpointSliceFromBackendServers(cur *discoveryv1.Endp
 			}
 		}
 		// add the non-existing endpoint
+		cond := false
 		if !existing {
 			endpoint := discoveryv1.Endpoint{
 				Addresses: []string{address},
@@ -651,7 +652,7 @@ func (m *Manager) constructEndpointSliceFromBackendServers(cur *discoveryv1.Endp
 					UID:       server.GetUID(),
 				},
 				Conditions: discoveryv1.EndpointConditions{
-					Ready: pointer.Bool(true),
+					Ready: &cond,
 				},
 			}
 			endpoints = append(endpoints, endpoint)
diff --git a/pkg/prober/manager_test.go b/pkg/prober/manager_test.go
index f9fd4101..d6a5b014 100644
--- a/pkg/prober/manager_test.go
+++ b/pkg/prober/manager_test.go
@@ -28,7 +28,7 @@ func TestManager(t *testing.T) {
 	}); err != nil {
 		t.Errorf("case: %s, add worker failed %s", healthyCase, err.Error())
 	}
-	time.Sleep(time.Second * 2)
+	time.Sleep(time.Second * 10)
 	if len(mng.workers) == 0 {
 		t.Errorf("case: %s, add worker failed", healthyCase)
 	}
@@ -61,7 +61,7 @@ func TestManager(t *testing.T) {
 	if len(mng.workers[unhealthyCase]) != 1 {
 		t.Errorf("case: %s, Add worker failed, len=%d", unhealthyCase, len(mng.workers[unhealthyCase]))
 	}
-	time.Sleep(time.Second * 5)
+	time.Sleep(time.Second * 10)
 	if mng.workers[unhealthyCase][unhealthyAddress].condition {
 		t.Errorf("it should not be able to connect %s", unhealthyAddress)
 	}