Skip to content

Commit 410c5d4

Browse files
zach593zach593
committed
use singleton ratelimiter for dynamically created clients
Signed-off-by: zach593 <[email protected]>
1 parent 8d43fe2 commit 410c5d4

File tree

20 files changed

+296
-101
lines changed

20 files changed

+296
-101
lines changed

‎cmd/agent/app/agent.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,10 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
265265
return fmt.Errorf("failed to setup custom resource interpreter: %w", err)
266266
}
267267

268-
objectWatcher := objectwatcher.NewObjectWatcher(mgr.GetClient(), mgr.GetRESTMapper(), util.NewClusterDynamicClientSetForAgent, resourceInterpreter)
268+
rateLimiterGetter := util.GetRateLimiterGetter().SetLimits(opts.ClusterAPIQPS, opts.ClusterAPIBurst)
269+
clusterClientOption := &util.ClientOption{RateLimiterGetter: rateLimiterGetter.GetRateLimiter}
270+
271+
objectWatcher := objectwatcher.NewObjectWatcher(mgr.GetClient(), mgr.GetRESTMapper(), util.NewClusterDynamicClientSetForAgent, clusterClientOption, resourceInterpreter)
269272
controllerContext := controllerscontext.Context{
270273
Mgr: mgr,
271274
ObjectWatcher: objectWatcher,
@@ -278,8 +281,6 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
278281
ClusterSuccessThreshold: opts.ClusterSuccessThreshold,
279282
ClusterFailureThreshold: opts.ClusterFailureThreshold,
280283
ClusterCacheSyncTimeout: opts.ClusterCacheSyncTimeout,
281-
ClusterAPIQPS: opts.ClusterAPIQPS,
282-
ClusterAPIBurst: opts.ClusterAPIBurst,
283284
ConcurrentWorkSyncs: opts.ConcurrentWorkSyncs,
284285
RateLimiterOptions: opts.RateLimiterOpts,
285286
EnableClusterResourceModeling: opts.EnableClusterResourceModeling,
@@ -289,6 +290,7 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
289290
},
290291
StopChan: stopChan,
291292
ResourceInterpreter: resourceInterpreter,
293+
ClusterClientOption: clusterClientOption,
292294
}
293295

294296
if err := controllers.StartControllers(controllerContext, controllersDisabledByDefault); err != nil {
@@ -315,7 +317,7 @@ func startClusterStatusController(ctx controllerscontext.Context) (bool, error)
315317
StopChan: ctx.StopChan,
316318
ClusterClientSetFunc: util.NewClusterClientSetForAgent,
317319
ClusterDynamicClientSetFunc: util.NewClusterDynamicClientSetForAgent,
318-
ClusterClientOption: &util.ClientOption{QPS: ctx.Opts.ClusterAPIQPS, Burst: ctx.Opts.ClusterAPIBurst},
320+
ClusterClientOption: ctx.ClusterClientOption,
319321
ClusterStatusUpdateFrequency: ctx.Opts.ClusterStatusUpdateFrequency,
320322
ClusterLeaseDuration: ctx.Opts.ClusterLeaseDuration,
321323
ClusterLeaseRenewIntervalFraction: ctx.Opts.ClusterLeaseRenewIntervalFraction,

‎cmd/controller-manager/app/controllermanager.go

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ func startClusterStatusController(ctx controllerscontext.Context) (enabled bool,
324324
StopChan: stopChan,
325325
ClusterClientSetFunc: util.NewClusterClientSet,
326326
ClusterDynamicClientSetFunc: util.NewClusterDynamicClientSet,
327-
ClusterClientOption: &util.ClientOption{QPS: opts.ClusterAPIQPS, Burst: opts.ClusterAPIBurst},
327+
ClusterClientOption: ctx.ClusterClientOption,
328328
ClusterStatusUpdateFrequency: opts.ClusterStatusUpdateFrequency,
329329
ClusterLeaseDuration: opts.ClusterLeaseDuration,
330330
ClusterLeaseRenewIntervalFraction: opts.ClusterLeaseRenewIntervalFraction,
@@ -428,6 +428,7 @@ func startWorkStatusController(ctx controllerscontext.Context) (enabled bool, er
428428
ObjectWatcher: ctx.ObjectWatcher,
429429
PredicateFunc: helper.NewExecutionPredicate(ctx.Mgr),
430430
ClusterDynamicClientSetFunc: util.NewClusterDynamicClientSet,
431+
ClusterClientOption: ctx.ClusterClientOption,
431432
ClusterCacheSyncTimeout: opts.ClusterCacheSyncTimeout,
432433
ConcurrentWorkStatusSyncs: opts.ConcurrentWorkSyncs,
433434
RateLimiterOptions: ctx.Opts.RateLimiterOptions,
@@ -465,6 +466,7 @@ func startServiceExportController(ctx controllerscontext.Context) (enabled bool,
465466
WorkerNumber: 3,
466467
PredicateFunc: helper.NewPredicateForServiceExportController(ctx.Mgr),
467468
ClusterDynamicClientSetFunc: util.NewClusterDynamicClientSet,
469+
ClusterClientOption: ctx.ClusterClientOption,
468470
ClusterCacheSyncTimeout: opts.ClusterCacheSyncTimeout,
469471
}
470472
serviceExportController.RunWorkQueue()
@@ -487,6 +489,7 @@ func startEndpointSliceCollectController(ctx controllerscontext.Context) (enable
487489
WorkerNumber: 3,
488490
PredicateFunc: helper.NewPredicateForEndpointSliceCollectController(ctx.Mgr),
489491
ClusterDynamicClientSetFunc: util.NewClusterDynamicClientSet,
492+
ClusterClientOption: ctx.ClusterClientOption,
490493
ClusterCacheSyncTimeout: opts.ClusterCacheSyncTimeout,
491494
}
492495
endpointSliceCollectController.RunWorkQueue()
@@ -744,6 +747,10 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
744747
}
745748

746749
controlPlaneInformerManager := genericmanager.NewSingleClusterInformerManager(dynamicClientSet, opts.ResyncPeriod.Duration, stopChan)
750+
751+
rateLimiterGetter := util.GetRateLimiterGetter().SetLimits(opts.ClusterAPIQPS, opts.ClusterAPIBurst)
752+
clusterClientOption := &util.ClientOption{RateLimiterGetter: rateLimiterGetter.GetRateLimiter}
753+
747754
// We need a service lister to build a resource interpreter with `ClusterIPServiceResolver`
748755
// witch allows connection to the customized interpreter webhook without a cluster DNS service.
749756
sharedFactory := informers.NewSharedInformerFactory(kubeClientSet, opts.ResyncPeriod.Duration)
@@ -756,7 +763,7 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
756763
klog.Fatalf("Failed to setup custom resource interpreter: %v", err)
757764
}
758765

759-
objectWatcher := objectwatcher.NewObjectWatcher(mgr.GetClient(), mgr.GetRESTMapper(), util.NewClusterDynamicClientSet, resourceInterpreter)
766+
objectWatcher := objectwatcher.NewObjectWatcher(mgr.GetClient(), mgr.GetRESTMapper(), util.NewClusterDynamicClientSet, clusterClientOption, resourceInterpreter)
760767

761768
resourceDetector := &detector.ResourceDetector{
762769
DiscoveryClientSet: discoverClientSet,
@@ -808,8 +815,6 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
808815
ClusterSuccessThreshold: opts.ClusterSuccessThreshold,
809816
ClusterFailureThreshold: opts.ClusterFailureThreshold,
810817
ClusterCacheSyncTimeout: opts.ClusterCacheSyncTimeout,
811-
ClusterAPIQPS: opts.ClusterAPIQPS,
812-
ClusterAPIBurst: opts.ClusterAPIBurst,
813818
SkippedPropagatingNamespaces: opts.SkippedNamespacesRegexps(),
814819
ConcurrentWorkSyncs: opts.ConcurrentWorkSyncs,
815820
EnableTaintManager: opts.EnableTaintManager,
@@ -824,6 +829,7 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
824829
OverrideManager: overrideManager,
825830
ControlPlaneInformerManager: controlPlaneInformerManager,
826831
ResourceInterpreter: resourceInterpreter,
832+
ClusterClientOption: clusterClientOption,
827833
}
828834

829835
if err := controllers.StartControllers(controllerContext, controllersDisabledByDefault); err != nil {

‎cmd/metrics-adapter/app/options/options.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,8 @@ func (o *Options) Config(stopCh <-chan struct{}) (*metricsadapter.MetricsServer,
134134
factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0)
135135
kubeClient := kubernetes.NewForConfigOrDie(restConfig)
136136
kubeFactory := informers.NewSharedInformerFactory(kubeClient, 0)
137-
metricsController := metricsadapter.NewMetricsController(stopCh, restConfig, factory, kubeFactory, &util.ClientOption{QPS: o.ClusterAPIQPS, Burst: o.ClusterAPIBurst})
137+
limiterGetter := util.GetRateLimiterGetter().SetLimits(o.ClusterAPIQPS, o.ClusterAPIBurst)
138+
metricsController := metricsadapter.NewMetricsController(stopCh, restConfig, factory, kubeFactory, &util.ClientOption{RateLimiterGetter: limiterGetter.GetRateLimiter})
138139
metricsAdapter := metricsadapter.NewMetricsAdapter(metricsController, o.CustomMetricsAdapterServerOptions)
139140
metricsAdapter.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(api.Scheme))
140141
metricsAdapter.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(api.Scheme))

‎pkg/controllers/context/context.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/karmada-io/karmada/pkg/controllers/federatedhpa/config"
3131
"github.com/karmada-io/karmada/pkg/resourceinterpreter"
3232
"github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag"
33+
"github.com/karmada-io/karmada/pkg/util"
3334
"github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager"
3435
"github.com/karmada-io/karmada/pkg/util/objectwatcher"
3536
"github.com/karmada-io/karmada/pkg/util/overridemanager"
@@ -67,10 +68,6 @@ type Options struct {
6768
ClusterFailureThreshold metav1.Duration
6869
// ClusterCacheSyncTimeout is the timeout period waiting for cluster cache to sync.
6970
ClusterCacheSyncTimeout metav1.Duration
70-
// ClusterAPIQPS is the QPS to use while talking with cluster kube-apiserver.
71-
ClusterAPIQPS float32
72-
// ClusterAPIBurst is the burst to allow while talking with cluster kube-apiserver.
73-
ClusterAPIBurst int
7471
// SkippedPropagatingNamespaces is a list of namespace regular expressions, matching namespaces will be skipped propagating.
7572
SkippedPropagatingNamespaces []*regexp.Regexp
7673
// ClusterName is the name of cluster.
@@ -112,6 +109,7 @@ type Context struct {
112109
OverrideManager overridemanager.OverrideManager
113110
ControlPlaneInformerManager genericmanager.SingleClusterInformerManager
114111
ResourceInterpreter resourceinterpreter.ResourceInterpreter
112+
ClusterClientOption *util.ClientOption
115113
}
116114

117115
// IsControllerEnabled check if a specified controller enabled or not.

‎pkg/controllers/execution/execution_controller_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ func newController(work *workv1alpha1.Work, recorder *record.FakeRecorder) Contr
229229
informerManager.ForCluster(cluster.Name, dynamicClientSet, 0).Lister(corev1.SchemeGroupVersion.WithResource("pods"))
230230
informerManager.Start(cluster.Name)
231231
informerManager.WaitForCacheSync(cluster.Name)
232-
clusterClientSetFunc := func(string, client.Client) (*util.DynamicClusterClient, error) {
232+
clusterClientSetFunc := func(string, client.Client, *util.ClientOption) (*util.DynamicClusterClient, error) {
233233
return &util.DynamicClusterClient{
234234
ClusterName: clusterName,
235235
DynamicClientSet: dynamicClientSet,
@@ -241,7 +241,7 @@ func newController(work *workv1alpha1.Work, recorder *record.FakeRecorder) Contr
241241
InformerManager: informerManager,
242242
EventRecorder: recorder,
243243
RESTMapper: restMapper,
244-
ObjectWatcher: objectwatcher.NewObjectWatcher(fakeClient, restMapper, clusterClientSetFunc, resourceInterpreter),
244+
ObjectWatcher: objectwatcher.NewObjectWatcher(fakeClient, restMapper, clusterClientSetFunc, nil, resourceInterpreter),
245245
}
246246
}
247247

‎pkg/controllers/mcs/service_export_controller.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ type ServiceExportController struct {
6868
InformerManager genericmanager.MultiClusterInformerManager
6969
WorkerNumber int // WorkerNumber is the number of worker goroutines
7070
PredicateFunc predicate.Predicate // PredicateFunc is the function that filters events before enqueuing the keys.
71-
ClusterDynamicClientSetFunc func(clusterName string, client client.Client) (*util.DynamicClusterClient, error)
71+
ClusterDynamicClientSetFunc util.NewClusterDynamicClientSetFunc
72+
ClusterClientOption *util.ClientOption
7273
ClusterCacheSyncTimeout metav1.Duration
7374

7475
// eventHandlers holds the handlers which used to handle events reported from member clusters.
@@ -234,7 +235,7 @@ func (c *ServiceExportController) buildResourceInformers(cluster *clusterv1alpha
234235
func (c *ServiceExportController) registerInformersAndStart(cluster *clusterv1alpha1.Cluster) error {
235236
singleClusterInformerManager := c.InformerManager.GetSingleClusterManager(cluster.Name)
236237
if singleClusterInformerManager == nil {
237-
dynamicClusterClient, err := c.ClusterDynamicClientSetFunc(cluster.Name, c.Client)
238+
dynamicClusterClient, err := c.ClusterDynamicClientSetFunc(cluster.Name, c.Client, c.ClusterClientOption)
238239
if err != nil {
239240
klog.Errorf("Failed to build dynamic cluster client for cluster %s.", cluster.Name)
240241
return err

‎pkg/controllers/multiclusterservice/endpointslice_collect_controller.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ type EndpointSliceCollectController struct {
6060
InformerManager genericmanager.MultiClusterInformerManager
6161
WorkerNumber int // WorkerNumber is the number of worker goroutines
6262
PredicateFunc predicate.Predicate // PredicateFunc is the function that filters events before enqueuing the keys.
63-
ClusterDynamicClientSetFunc func(clusterName string, client client.Client) (*util.DynamicClusterClient, error)
63+
ClusterDynamicClientSetFunc util.NewClusterDynamicClientSetFunc
64+
ClusterClientOption *util.ClientOption
6465
// eventHandlers holds the handlers which used to handle events reported from member clusters.
6566
// Each handler takes the cluster name as key and takes the handler function as the value, e.g.
6667
// "member1": instance of ResourceEventHandler
@@ -177,7 +178,7 @@ func (c *EndpointSliceCollectController) buildResourceInformers(clusterName stri
177178
func (c *EndpointSliceCollectController) registerInformersAndStart(cluster *clusterv1alpha1.Cluster) error {
178179
singleClusterInformerManager := c.InformerManager.GetSingleClusterManager(cluster.Name)
179180
if singleClusterInformerManager == nil {
180-
dynamicClusterClient, err := c.ClusterDynamicClientSetFunc(cluster.Name, c.Client)
181+
dynamicClusterClient, err := c.ClusterDynamicClientSetFunc(cluster.Name, c.Client, c.ClusterClientOption)
181182
if err != nil {
182183
klog.Errorf("Failed to build dynamic cluster client for cluster %s.", cluster.Name)
183184
return err

‎pkg/controllers/status/cluster_status_controller.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ type ClusterStatusController struct {
8787
TypedInformerManager typedmanager.MultiClusterInformerManager
8888
GenericInformerManager genericmanager.MultiClusterInformerManager
8989
StopChan <-chan struct{}
90-
ClusterClientSetFunc func(string, client.Client, *util.ClientOption) (*util.ClusterClient, error)
91-
ClusterDynamicClientSetFunc func(clusterName string, client client.Client) (*util.DynamicClusterClient, error)
90+
ClusterClientSetFunc util.NewClusterClientSetFunc
91+
ClusterDynamicClientSetFunc util.NewClusterDynamicClientSetFunc
9292
// ClusterClientOption holds the attributes that should be injected to a Kubernetes client.
9393
ClusterClientOption *util.ClientOption
9494

@@ -343,7 +343,7 @@ func (c *ClusterStatusController) initializeGenericInformerManagerForCluster(clu
343343
return
344344
}
345345

346-
dynamicClient, err := c.ClusterDynamicClientSetFunc(clusterClient.ClusterName, c.Client)
346+
dynamicClient, err := c.ClusterDynamicClientSetFunc(clusterClient.ClusterName, c.Client, c.ClusterClientOption)
347347
if err != nil {
348348
klog.Errorf("Failed to build dynamic cluster client for cluster %s.", clusterClient.ClusterName)
349349
return

‎pkg/controllers/status/cluster_status_controller_test.go

Lines changed: 19 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,8 @@ func TestClusterStatusController_Reconcile(t *testing.T) {
113113
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).Build(),
114114
GenericInformerManager: genericmanager.GetInstance(),
115115
TypedInformerManager: typedmanager.GetInstance(),
116-
ClusterClientOption: &util.ClientOption{
117-
QPS: 5,
118-
Burst: 10,
119-
},
120-
ClusterClientSetFunc: util.NewClusterClientSet,
116+
ClusterClientOption: &util.ClientOption{},
117+
ClusterClientSetFunc: util.NewClusterClientSet,
121118
}
122119

123120
if tt.cluster != nil {
@@ -900,11 +897,8 @@ func TestClusterStatusController_updateStatusIfNeeded(t *testing.T) {
900897
).WithStatusSubresource(cluster).Build(),
901898
GenericInformerManager: genericmanager.GetInstance(),
902899
TypedInformerManager: typedmanager.GetInstance(),
903-
ClusterClientOption: &util.ClientOption{
904-
QPS: 5,
905-
Burst: 10,
906-
},
907-
ClusterClientSetFunc: util.NewClusterClientSet,
900+
ClusterClientOption: &util.ClientOption{},
901+
ClusterClientSetFunc: util.NewClusterClientSet,
908902
}
909903

910904
err := c.updateStatusIfNeeded(context.Background(), cluster, currentClusterStatus)
@@ -964,32 +958,26 @@ func TestClusterStatusController_updateStatusIfNeeded(t *testing.T) {
964958
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).Build(),
965959
GenericInformerManager: genericmanager.GetInstance(),
966960
TypedInformerManager: typedmanager.GetInstance(),
967-
ClusterClientOption: &util.ClientOption{
968-
QPS: 5,
969-
Burst: 10,
970-
},
971-
ClusterClientSetFunc: util.NewClusterClientSet,
961+
ClusterClientOption: &util.ClientOption{},
962+
ClusterClientSetFunc: util.NewClusterClientSet,
972963
}
973964

974965
err := c.updateStatusIfNeeded(context.Background(), cluster, currentClusterStatus)
975966
assert.NotEmpty(t, err, "updateStatusIfNeeded doesn't return error")
976967
})
977968
}
978969

979-
func NewClusterDynamicClientSetForAgentWithError(_ string, _ client.Client) (*util.DynamicClusterClient, error) {
970+
func NewClusterDynamicClientSetForAgentWithError(_ string, _ client.Client, _ *util.ClientOption) (*util.DynamicClusterClient, error) {
980971
return nil, fmt.Errorf("err")
981972
}
982973

983974
func TestClusterStatusController_initializeGenericInformerManagerForCluster(t *testing.T) {
984975
t.Run("failed to create dynamicClient", func(*testing.T) {
985976
c := &ClusterStatusController{
986-
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).Build(),
987-
GenericInformerManager: genericmanager.GetInstance(),
988-
TypedInformerManager: typedmanager.GetInstance(),
989-
ClusterClientOption: &util.ClientOption{
990-
QPS: 5,
991-
Burst: 10,
992-
},
977+
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).Build(),
978+
GenericInformerManager: genericmanager.GetInstance(),
979+
TypedInformerManager: typedmanager.GetInstance(),
980+
ClusterClientOption: &util.ClientOption{},
993981
ClusterClientSetFunc: util.NewClusterClientSet,
994982
ClusterDynamicClientSetFunc: NewClusterDynamicClientSetForAgentWithError,
995983
}
@@ -1002,13 +990,10 @@ func TestClusterStatusController_initializeGenericInformerManagerForCluster(t *t
1002990

1003991
t.Run("suc to create dynamicClient", func(*testing.T) {
1004992
c := &ClusterStatusController{
1005-
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).Build(),
1006-
GenericInformerManager: genericmanager.GetInstance(),
1007-
TypedInformerManager: typedmanager.GetInstance(),
1008-
ClusterClientOption: &util.ClientOption{
1009-
QPS: 5,
1010-
Burst: 10,
1011-
},
993+
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).Build(),
994+
GenericInformerManager: genericmanager.GetInstance(),
995+
TypedInformerManager: typedmanager.GetInstance(),
996+
ClusterClientOption: &util.ClientOption{},
1012997
ClusterClientSetFunc: util.NewClusterClientSet,
1013998
ClusterDynamicClientSetFunc: util.NewClusterDynamicClientSetForAgent,
1014999
}
@@ -1022,13 +1007,10 @@ func TestClusterStatusController_initializeGenericInformerManagerForCluster(t *t
10221007

10231008
func TestClusterStatusController_initLeaseController(_ *testing.T) {
10241009
c := &ClusterStatusController{
1025-
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).Build(),
1026-
GenericInformerManager: genericmanager.GetInstance(),
1027-
TypedInformerManager: typedmanager.GetInstance(),
1028-
ClusterClientOption: &util.ClientOption{
1029-
QPS: 5,
1030-
Burst: 10,
1031-
},
1010+
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).Build(),
1011+
GenericInformerManager: genericmanager.GetInstance(),
1012+
TypedInformerManager: typedmanager.GetInstance(),
1013+
ClusterClientOption: &util.ClientOption{},
10321014
ClusterClientSetFunc: util.NewClusterClientSet,
10331015
ClusterDynamicClientSetFunc: NewClusterDynamicClientSetForAgentWithError,
10341016
}

0 commit comments

Comments
 (0)