From a5c93dae6feb35e7ba48271e05eacf4a0c522709 Mon Sep 17 00:00:00 2001 From: "R.K" Date: Tue, 4 Feb 2025 11:57:07 +0200 Subject: [PATCH] Add configurable QPS and burst settings for kube API client Introduce new flags to configure `QPS` and `Burst` for the Kubernetes API client, enabling better control over API rate limits. Signed-off-by: R.K --- cmd/training-operator.v1/main.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/training-operator.v1/main.go b/cmd/training-operator.v1/main.go index 41a28ff590..805aaf70a1 100644 --- a/cmd/training-operator.v1/main.go +++ b/cmd/training-operator.v1/main.go @@ -81,6 +81,8 @@ func main() { var webhookServerPort int var webhookServiceName string var webhookSecretName string + var clientQps int + var clientBurst int flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -95,7 +97,8 @@ func main() { flag.StringVar(&namespace, "namespace", os.Getenv(EnvKubeflowNamespace), "The namespace to monitor kubeflow jobs. If unset, it monitors all namespaces cluster-wide."+ "If set, it only monitors kubeflow jobs in the given namespace.") flag.IntVar(&controllerThreads, "controller-threads", 1, "Number of worker threads used by the controller.") - + flag.IntVar(&clientQps, "kube-api-qps", 20, "QPS indicates the maximum QPS to the master from this client.") + flag.IntVar(&clientBurst, "kube-api-burst", 30, "Maximum burst for throttle.") // PyTorch related flags flag.StringVar(&config.Config.PyTorchInitContainerImage, "pytorch-init-container-image", config.PyTorchInitContainerImageDefault, "The image for pytorch init container") @@ -131,7 +134,11 @@ func main() { } } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + cfg := ctrl.GetConfigOrDie() + cfg.QPS = float32(clientQps) + cfg.Burst = clientBurst + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ BindAddress: metricsAddr,