Adds tolerations from fleet-controller when bootstraping and when run…

…ning the fleet apply job Adding the toleration needed to the helm chart in Fleet is not enough when running the agent and the fleet apply job. This PR adds the tolerations found in the `fleet-controller` deployment to the agent and to the fleet apply job. Refers to: rancher#3313 Signed-off-by: Xavi Garcia <[email protected]>
0xavi0 · Feb 17, 2025 · d3d869d · d3d869d
1 parent ff036eb
commit d3d869d
Show file tree

Hide file tree

Showing 7 changed files with 244 additions and 32 deletions.
diff --git a/charts/fleet/templates/rbac.yaml b/charts/fleet/templates/rbac.yaml
@@ -38,6 +38,13 @@ rules:
     - 'events'
   verbs:
     - '*'
+- apiGroups:
+    - "apps"
+  resources:
+    - 'deployments'
+  verbs:
+    - 'list'
+    - 'get'
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/integrationtests/gitjob/controller/suite_test.go b/integrationtests/gitjob/controller/suite_test.go
@@ -14,6 +14,8 @@ import (
 	gomegatypes "github.com/onsi/gomega/types"
 	"github.com/reugn/go-quartz/quartz"
 	"go.uber.org/mock/gomock"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 
 	"github.com/rancher/fleet/internal/cmd/controller/gitops/reconciler"
 	ctrlreconciler "github.com/rancher/fleet/internal/cmd/controller/reconciler"
@@ -99,20 +101,53 @@ var _ = BeforeSuite(func() {
 		},
 	)
 
+	// fleet-controller deployment
+	err = k8sClient.Create(ctx, &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      config.ManagerConfigName,
+			Namespace: "default",
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": "fleet-controller",
+				},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app": "fleet-controller",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "test",
+							Image: "test", // value is required. but we don't need a real deployment for the test
+
+						},
+					},
+				},
+			},
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+
 	sched := quartz.NewStdScheduler()
 	Expect(sched).ToNot(BeNil())
 
 	config.Set(&config.Config{})
 
 	err = (&reconciler.GitJobReconciler{
-		Client:     mgr.GetClient(),
-		Scheme:     mgr.GetScheme(),
-		Image:      "image",
-		Scheduler:  sched,
-		GitFetcher: fetcherMock,
-		Clock:      reconciler.RealClock{},
-		Recorder:   mgr.GetEventRecorderFor("gitjob-controller"),
-		Workers:    50,
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		Image:           "image",
+		Scheduler:       sched,
+		GitFetcher:      fetcherMock,
+		Clock:           reconciler.RealClock{},
+		Recorder:        mgr.GetEventRecorderFor("gitjob-controller"),
+		Workers:         50,
+		SystemNamespace: "default",
 	}).SetupWithManager(mgr)
 	Expect(err).ToNot(HaveOccurred())
 

diff --git a/internal/cmd/controller/agentmanagement/controllers/bootstrap/bootstrap.go b/internal/cmd/controller/agentmanagement/controllers/bootstrap/bootstrap.go
@@ -10,14 +10,16 @@ import (
 
 	secretutil "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/secret"
 	fleetns "github.com/rancher/fleet/internal/cmd/controller/namespace"
-	"github.com/rancher/fleet/internal/config"
+	fleetconfig "github.com/rancher/fleet/internal/config"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 	"github.com/rancher/wrangler/v3/pkg/apply"
+	appscontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1"
 	corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -39,6 +41,7 @@ type handler struct {
 	serviceAccountCache corecontrollers.ServiceAccountCache
 	secretsCache        corecontrollers.SecretCache
 	secretsController   corecontrollers.SecretController
+	deploymentsCache    appscontrollers.DeploymentCache
 	cfg                 clientcmd.ClientConfig
 }
 
@@ -49,19 +52,21 @@ func Register(ctx context.Context,
 	serviceAccountCache corecontrollers.ServiceAccountCache,
 	secretsController corecontrollers.SecretController,
 	secretsCache corecontrollers.SecretCache,
+	deploymentCache appscontrollers.DeploymentCache,
 ) {
 	h := handler{
 		systemNamespace:     systemNamespace,
 		serviceAccountCache: serviceAccountCache,
 		secretsCache:        secretsCache,
 		secretsController:   secretsController,
+		deploymentsCache:    deploymentCache,
 		apply:               apply.WithSetID("fleet-bootstrap"),
 		cfg:                 cfg,
 	}
-	config.OnChange(ctx, h.OnConfig)
+	fleetconfig.OnChange(ctx, h.OnConfig)
 }
 
-func (h *handler) OnConfig(config *config.Config) error {
+func (h *handler) OnConfig(config *fleetconfig.Config) error {
 	logrus.Debugf("Bootstrap config set, building namespace '%s', secret, local cluster, cluster group, ...", config.Bootstrap.Namespace)
 
 	var objs []runtime.Object
@@ -74,6 +79,10 @@ func (h *handler) OnConfig(config *config.Config) error {
 	if err != nil {
 		return err
 	}
+	fleetControllerDeployment, err := h.deploymentsCache.Get(h.systemNamespace, fleetconfig.ManagerConfigName)
+	if err != nil {
+		return err
+	}
 	objs = append(objs, &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: config.Bootstrap.Namespace,
@@ -89,6 +98,8 @@ func (h *handler) OnConfig(config *config.Config) error {
 		Spec: fleet.ClusterSpec{
 			KubeConfigSecret: secret.Name,
 			AgentNamespace:   config.Bootstrap.AgentNamespace,
+			// copy tolerations from fleet-controller
+			AgentTolerations: fleetControllerDeployment.Spec.Template.Spec.Tolerations,
 		},
 	}, &fleet.ClusterGroup{
 		ObjectMeta: metav1.ObjectMeta{
@@ -164,9 +175,9 @@ func (h *handler) buildSecret(bootstrapNamespace string, cfg clientcmd.ClientCon
 			},
 		},
 		Data: map[string][]byte{
-			config.KubeConfigSecretValueKey: value,
-			config.APIServerURLKey:          []byte(host),
-			config.APIServerCAKey:           ca,
+			fleetconfig.KubeConfigSecretValueKey: value,
+			fleetconfig.APIServerURLKey:          []byte(host),
+			fleetconfig.APIServerCAKey:           ca,
 		},
 	}, nil
 }

diff --git a/internal/cmd/controller/agentmanagement/controllers/controllers.go b/internal/cmd/controller/agentmanagement/controllers/controllers.go
@@ -20,6 +20,8 @@ import (
 	"github.com/rancher/lasso/pkg/client"
 	"github.com/rancher/lasso/pkg/controller"
 	"github.com/rancher/wrangler/v3/pkg/apply"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/apps"
+	appscontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1"
 	"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
 	corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/v3/pkg/generated/controllers/rbac"
@@ -42,6 +44,7 @@ type AppContext struct {
 
 	K8s          kubernetes.Interface
 	Core         corecontrollers.Interface
+	Apps         appscontrollers.Interface
 	RBAC         rbaccontrollers.Interface
 	RESTMapper   meta.RESTMapper
 	Apply        apply.Apply
@@ -86,7 +89,8 @@ func Register(ctx context.Context, appCtx *AppContext, systemNamespace string, d
 			appCtx.ClientConfig,
 			appCtx.Core.ServiceAccount().Cache(),
 			appCtx.Core.Secret(),
-			appCtx.Core.Secret().Cache())
+			appCtx.Core.Secret().Cache(),
+			appCtx.Apps.Deployment().Cache())
 	}
 
 	cluster.Register(ctx,
@@ -182,6 +186,14 @@ func NewAppContext(cfg clientcmd.ClientConfig) (*AppContext, error) {
 	}
 	rbacv := rbac.Rbac().V1()
 
+	apps, err := apps.NewFactoryFromConfigWithOptions(client, &apps.FactoryOptions{
+		SharedControllerFactory: scf,
+	})
+	if err != nil {
+		return nil, err
+	}
+	appsv := apps.Apps().V1()
+
 	apply, err := apply.NewForConfig(client)
 	if err != nil {
 		return nil, err
@@ -197,6 +209,7 @@ func NewAppContext(cfg clientcmd.ClientConfig) (*AppContext, error) {
 		K8s:          k8s,
 		Interface:    fleetv,
 		Core:         corev,
+		Apps:         appsv,
 		RBAC:         rbacv,
 		Apply:        apply,
 		ClientConfig: cfg,

diff --git a/internal/cmd/controller/gitops/operator.go b/internal/cmd/controller/gitops/operator.go
@@ -144,6 +144,7 @@ func (g *GitOperator) Run(cmd *cobra.Command, args []string) error {
 		GitFetcher:      &git.Fetch{},
 		Clock:           reconciler.RealClock{},
 		Recorder:        mgr.GetEventRecorderFor(fmt.Sprintf("fleet-gitops%s", shardIDSuffix)),
+		SystemNamespace: namespace,
 	}
 
 	statusReconciler := &reconciler.StatusReconciler{

diff --git a/internal/cmd/controller/gitops/reconciler/gitjob_controller.go b/internal/cmd/controller/gitops/reconciler/gitjob_controller.go
@@ -18,6 +18,7 @@ import (
 	fleetutil "github.com/rancher/fleet/internal/cmd/controller/errorutil"
 	"github.com/rancher/fleet/internal/cmd/controller/finalize"
 	"github.com/rancher/fleet/internal/cmd/controller/imagescan"
+	"github.com/rancher/fleet/internal/config"
 	"github.com/rancher/fleet/internal/metrics"
 	"github.com/rancher/fleet/internal/names"
 	"github.com/rancher/fleet/internal/ociwrapper"
@@ -29,6 +30,7 @@ import (
 	"github.com/rancher/wrangler/v3/pkg/genericcondition"
 	"github.com/rancher/wrangler/v3/pkg/kstatus"
 
+	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -97,6 +99,7 @@ type GitJobReconciler struct {
 	GitFetcher      GitFetcher
 	Clock           TimeGetter
 	Recorder        record.EventRecorder
+	SystemNamespace string
 }
 
 func (r *GitJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
@@ -535,7 +538,19 @@ func (r *GitJobReconciler) newGitJob(ctx context.Context, obj *v1alpha1.GitRepo)
 	if err != nil {
 		return nil, err
 	}
+	var fleetControllerDeployment appsv1.Deployment
+	if err := r.Get(ctx, types.NamespacedName{
+		Namespace: r.SystemNamespace,
+		Name:      config.ManagerConfigName,
+	}, &fleetControllerDeployment); err != nil {
+		return nil, err
+	}
 
+	// add tolerations from the fleet-controller deployment
+	jobSpec.Template.Spec.Tolerations = append(
+		jobSpec.Template.Spec.Tolerations,
+		fleetControllerDeployment.Spec.Template.Spec.Tolerations...,
+	)
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Annotations: map[string]string{