diff --git a/charts/fleet/templates/rbac.yaml b/charts/fleet/templates/rbac.yaml index 026c1c26d2..8fce7359f6 100644 --- a/charts/fleet/templates/rbac.yaml +++ b/charts/fleet/templates/rbac.yaml @@ -38,6 +38,13 @@ rules: - 'events' verbs: - '*' +- apiGroups: + - "apps" + resources: + - 'deployments' + verbs: + - 'list' + - 'get' --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/charts/fleet/templates/rbac_gitjob.yaml b/charts/fleet/templates/rbac_gitjob.yaml index e2c93169de..7104ad8b6a 100644 --- a/charts/fleet/templates/rbac_gitjob.yaml +++ b/charts/fleet/templates/rbac_gitjob.yaml @@ -91,7 +91,14 @@ rules: - rolebindings verbs: - create - + - apiGroups: + - "apps" + resources: + - 'deployments' + verbs: + - 'list' + - 'get' + - 'watch' --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/integrationtests/gitjob/controller/suite_test.go b/integrationtests/gitjob/controller/suite_test.go index 1935e5d0b7..2cbb9c77c8 100644 --- a/integrationtests/gitjob/controller/suite_test.go +++ b/integrationtests/gitjob/controller/suite_test.go @@ -14,6 +14,8 @@ import ( gomegatypes "github.com/onsi/gomega/types" "github.com/reugn/go-quartz/quartz" "go.uber.org/mock/gomock" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "github.com/rancher/fleet/internal/cmd/controller/gitops/reconciler" ctrlreconciler "github.com/rancher/fleet/internal/cmd/controller/reconciler" @@ -99,19 +101,53 @@ var _ = BeforeSuite(func() { }, ) + // fleet-controller deployment + err = k8sClient.Create(ctx, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.ManagerConfigName, + Namespace: "default", + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "fleet-controller", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "fleet-controller", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "test", // value is required. but we don't need a real deployment for the test + + }, + }, + }, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + sched := quartz.NewStdScheduler() Expect(sched).ToNot(BeNil()) config.Set(&config.Config{}) err = (&reconciler.GitJobReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Image: "image", - Scheduler: sched, - GitFetcher: fetcherMock, - Clock: reconciler.RealClock{}, - Recorder: mgr.GetEventRecorderFor("gitjob-controller"), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Image: "image", + Scheduler: sched, + GitFetcher: fetcherMock, + Clock: reconciler.RealClock{}, + Recorder: mgr.GetEventRecorderFor("gitjob-controller"), + Workers: 50, + SystemNamespace: "default", }).SetupWithManager(mgr) Expect(err).ToNot(HaveOccurred()) diff --git a/internal/cmd/controller/agentmanagement/controllers/bootstrap/bootstrap.go b/internal/cmd/controller/agentmanagement/controllers/bootstrap/bootstrap.go index 9e25fde239..8a8f13d3e7 100644 --- a/internal/cmd/controller/agentmanagement/controllers/bootstrap/bootstrap.go +++ b/internal/cmd/controller/agentmanagement/controllers/bootstrap/bootstrap.go @@ -10,14 +10,16 @@ import ( secretutil "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/secret" fleetns "github.com/rancher/fleet/internal/cmd/controller/namespace" - "github.com/rancher/fleet/internal/config" + fleetconfig "github.com/rancher/fleet/internal/config" fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1" "github.com/rancher/wrangler/v3/pkg/apply" + appscontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1" corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -39,6 +41,7 @@ type handler struct { serviceAccountCache corecontrollers.ServiceAccountCache secretsCache corecontrollers.SecretCache secretsController corecontrollers.SecretController + deploymentsCache appscontrollers.DeploymentCache cfg clientcmd.ClientConfig } @@ -49,19 +52,21 @@ func Register(ctx context.Context, serviceAccountCache corecontrollers.ServiceAccountCache, secretsController corecontrollers.SecretController, secretsCache corecontrollers.SecretCache, + deploymentCache appscontrollers.DeploymentCache, ) { h := handler{ systemNamespace: systemNamespace, serviceAccountCache: serviceAccountCache, secretsCache: secretsCache, secretsController: secretsController, + deploymentsCache: deploymentCache, apply: apply.WithSetID("fleet-bootstrap"), cfg: cfg, } - config.OnChange(ctx, h.OnConfig) + fleetconfig.OnChange(ctx, h.OnConfig) } -func (h *handler) OnConfig(config *config.Config) error { +func (h *handler) OnConfig(config *fleetconfig.Config) error { logrus.Debugf("Bootstrap config set, building namespace '%s', secret, local cluster, cluster group, ...", config.Bootstrap.Namespace) var objs []runtime.Object @@ -74,6 +79,10 @@ func (h *handler) OnConfig(config *config.Config) error { if err != nil { return err } + fleetControllerDeployment, err := h.deploymentsCache.Get(h.systemNamespace, fleetconfig.ManagerConfigName) + if err != nil { + return err + } objs = append(objs, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: config.Bootstrap.Namespace, @@ -89,6 +98,8 @@ func (h *handler) OnConfig(config *config.Config) error { Spec: fleet.ClusterSpec{ KubeConfigSecret: secret.Name, AgentNamespace: config.Bootstrap.AgentNamespace, + // copy tolerations from fleet-controller + AgentTolerations: fleetControllerDeployment.Spec.Template.Spec.Tolerations, }, }, &fleet.ClusterGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -164,9 +175,9 @@ func (h *handler) buildSecret(bootstrapNamespace string, cfg clientcmd.ClientCon }, }, Data: map[string][]byte{ - config.KubeConfigSecretValueKey: value, - config.APIServerURLKey: []byte(host), - config.APIServerCAKey: ca, + fleetconfig.KubeConfigSecretValueKey: value, + fleetconfig.APIServerURLKey: []byte(host), + fleetconfig.APIServerCAKey: ca, }, }, nil } diff --git a/internal/cmd/controller/agentmanagement/controllers/controllers.go b/internal/cmd/controller/agentmanagement/controllers/controllers.go index be60e09fef..b67641e7cc 100644 --- a/internal/cmd/controller/agentmanagement/controllers/controllers.go +++ b/internal/cmd/controller/agentmanagement/controllers/controllers.go @@ -20,6 +20,8 @@ import ( "github.com/rancher/lasso/pkg/client" "github.com/rancher/lasso/pkg/controller" "github.com/rancher/wrangler/v3/pkg/apply" + "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps" + appscontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1" "github.com/rancher/wrangler/v3/pkg/generated/controllers/core" corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1" "github.com/rancher/wrangler/v3/pkg/generated/controllers/rbac" @@ -42,6 +44,7 @@ type AppContext struct { K8s kubernetes.Interface Core corecontrollers.Interface + Apps appscontrollers.Interface RBAC rbaccontrollers.Interface RESTMapper meta.RESTMapper Apply apply.Apply @@ -86,7 +89,8 @@ func Register(ctx context.Context, appCtx *AppContext, systemNamespace string, d appCtx.ClientConfig, appCtx.Core.ServiceAccount().Cache(), appCtx.Core.Secret(), - appCtx.Core.Secret().Cache()) + appCtx.Core.Secret().Cache(), + appCtx.Apps.Deployment().Cache()) } cluster.Register(ctx, @@ -182,6 +186,14 @@ func NewAppContext(cfg clientcmd.ClientConfig) (*AppContext, error) { } rbacv := rbac.Rbac().V1() + apps, err := apps.NewFactoryFromConfigWithOptions(client, &apps.FactoryOptions{ + SharedControllerFactory: scf, + }) + if err != nil { + return nil, err + } + appsv := apps.Apps().V1() + apply, err := apply.NewForConfig(client) if err != nil { return nil, err @@ -197,6 +209,7 @@ func NewAppContext(cfg clientcmd.ClientConfig) (*AppContext, error) { K8s: k8s, Interface: fleetv, Core: corev, + Apps: appsv, RBAC: rbacv, Apply: apply, ClientConfig: cfg, diff --git a/internal/cmd/controller/gitops/operator.go b/internal/cmd/controller/gitops/operator.go index 60d0cf03bb..71af2a0805 100644 --- a/internal/cmd/controller/gitops/operator.go +++ b/internal/cmd/controller/gitops/operator.go @@ -146,6 +146,7 @@ func (g *GitOperator) Run(cmd *cobra.Command, args []string) error { GitFetcher: &git.Fetch{}, Clock: reconciler.RealClock{}, Recorder: mgr.GetEventRecorderFor(fmt.Sprintf("fleet-gitops%s", shardIDSuffix)), + SystemNamespace: namespace, } statusReconciler := &reconciler.StatusReconciler{ diff --git a/internal/cmd/controller/gitops/reconciler/gitjob_controller.go b/internal/cmd/controller/gitops/reconciler/gitjob_controller.go index 2b799b2491..3510568388 100644 --- a/internal/cmd/controller/gitops/reconciler/gitjob_controller.go +++ b/internal/cmd/controller/gitops/reconciler/gitjob_controller.go @@ -18,6 +18,7 @@ import ( fleetutil "github.com/rancher/fleet/internal/cmd/controller/errorutil" "github.com/rancher/fleet/internal/cmd/controller/finalize" "github.com/rancher/fleet/internal/cmd/controller/imagescan" + "github.com/rancher/fleet/internal/config" "github.com/rancher/fleet/internal/metrics" "github.com/rancher/fleet/internal/names" "github.com/rancher/fleet/internal/ociwrapper" @@ -29,6 +30,7 @@ import ( "github.com/rancher/wrangler/v3/pkg/genericcondition" "github.com/rancher/wrangler/v3/pkg/kstatus" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -97,6 +99,7 @@ type GitJobReconciler struct { GitFetcher GitFetcher Clock TimeGetter Recorder record.EventRecorder + SystemNamespace string } func (r *GitJobReconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -547,7 +550,19 @@ func (r *GitJobReconciler) newGitJob(ctx context.Context, obj *v1alpha1.GitRepo) if err != nil { return nil, err } + var fleetControllerDeployment appsv1.Deployment + if err := r.Get(ctx, types.NamespacedName{ + Namespace: r.SystemNamespace, + Name: config.ManagerConfigName, + }, &fleetControllerDeployment); err != nil { + return nil, err + } + // add tolerations from the fleet-controller deployment + jobSpec.Template.Spec.Tolerations = append( + jobSpec.Template.Spec.Tolerations, + fleetControllerDeployment.Spec.Template.Spec.Tolerations..., + ) job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ diff --git a/internal/cmd/controller/gitops/reconciler/gitjob_test.go b/internal/cmd/controller/gitops/reconciler/gitjob_test.go index d4dcf720ea..3e534dc2de 100644 --- a/internal/cmd/controller/gitops/reconciler/gitjob_test.go +++ b/internal/cmd/controller/gitops/reconciler/gitjob_test.go @@ -14,12 +14,14 @@ import ( "go.uber.org/mock/gomock" "github.com/rancher/fleet/internal/cmd/controller/finalize" + "github.com/rancher/fleet/internal/config" "github.com/rancher/fleet/internal/mocks" fleetv1 "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1" "github.com/rancher/wrangler/v3/pkg/genericcondition" fleetevent "github.com/rancher/fleet/pkg/event" gitmocks "github.com/rancher/fleet/pkg/git/mocks" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -34,6 +36,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) +type testClientType int + +const ( + testClientHTTP testClientType = iota + testClientSSH + testClientSimple +) + func getCondition(gitrepo *fleetv1.GitRepo, condType string) (genericcondition.GenericCondition, bool) { for _, cond := range gitrepo.Status.Conditions { if cond.Type == condType { @@ -677,9 +687,25 @@ func TestNewJob(t *testing.T) { // nolint:funlen utilruntime.Must(batchv1.AddToScheme(scheme)) ctx := context.TODO() + // define the default tolerations that all jobs have + defaultTolerations := []corev1.Toleration{ + { + Key: "cattle.io/os", + Operator: "Equal", + Value: "linux", + Effect: "NoSchedule", + }, + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Operator: "Equal", + Value: "true", + Effect: "NoSchedule", + }, + } tests := map[string]struct { gitrepo *fleetv1.GitRepo - client client.Client + client testClientType + deploymentTolerations []corev1.Toleration expectedInitContainers []corev1.Container expectedVolumes []corev1.Volume expectedErr error @@ -723,7 +749,7 @@ func TestNewJob(t *testing.T) { // nolint:funlen }, }, }, - client: fake.NewFakeClient(), + client: testClientSimple, }, "simple with custom branch": { gitrepo: &fleetv1.GitRepo{ @@ -767,7 +793,7 @@ func TestNewJob(t *testing.T) { // nolint:funlen }, }, }, - client: fake.NewFakeClient(), + client: testClientSimple, }, "simple with custom revision": { gitrepo: &fleetv1.GitRepo{ @@ -811,7 +837,7 @@ func TestNewJob(t *testing.T) { // nolint:funlen }, }, }, - client: fake.NewFakeClient(), + client: testClientSimple, }, "http credentials": { gitrepo: &fleetv1.GitRepo{ @@ -877,7 +903,7 @@ func TestNewJob(t *testing.T) { // nolint:funlen }, }, }, - client: httpSecretMock(), + client: testClientHTTP, }, "ssh credentials": { gitrepo: &fleetv1.GitRepo{ @@ -941,7 +967,7 @@ func TestNewJob(t *testing.T) { // nolint:funlen }, }, }, - client: sshSecretMock(), + client: testClientSSH, }, "custom CA": { gitrepo: &fleetv1.GitRepo{ @@ -1056,16 +1082,73 @@ func TestNewJob(t *testing.T) { // nolint:funlen }, }, }, + "simple with tolerations": { + gitrepo: &fleetv1.GitRepo{ + Spec: fleetv1.GitRepoSpec{Repo: "repo"}, + }, + expectedInitContainers: []corev1.Container{ + { + Command: []string{ + "fleet", + }, + Args: []string{"gitcloner", "repo", "/workspace", "--branch", "master"}, + Image: "test", + Name: "gitcloner-initializer", + VolumeMounts: []corev1.VolumeMount{ + { + Name: gitClonerVolumeName, + MountPath: "/workspace", + }, + { + Name: emptyDirVolumeName, + MountPath: "/tmp", + }, + }, + SecurityContext: securityContext, + }, + }, + expectedVolumes: []corev1.Volume{ + { + Name: gitClonerVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: emptyDirVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + client: testClientSimple, + deploymentTolerations: []corev1.Toleration{ + { + Effect: "NoSchedule", + Key: "key1", + Value: "value1", + Operator: "Equals", + }, + { + Effect: "NoExecute", + Key: "key2", + Value: "value2", + Operator: "Exists", + }, + }, + }, } for name, test := range tests { t.Run(name, func(t *testing.T) { r := GitJobReconciler{ - Client: test.client, - Scheme: scheme, - Image: "test", - Clock: RealClock{}, + Client: getTestClient(test.client, test.deploymentTolerations), + Scheme: scheme, + Image: "test", + Clock: RealClock{}, + SystemNamespace: config.DefaultNamespace, } + job, err := r.newGitJob(ctx, test.gitrepo) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -1086,6 +1169,13 @@ func TestNewJob(t *testing.T) { // nolint:funlen t.Fatalf("volume %v not found in %v", evol, job.Spec.Template.Spec.Volumes) } } + + // tolerations check + // tolerations will be the default ones plus the deployment ones + expectedTolerations := append(defaultTolerations, test.deploymentTolerations...) + if !cmp.Equal(expectedTolerations, job.Spec.Template.Spec.Tolerations) { + t.Fatalf("job tolerations differ. Expecting: %v and found: %v", test.deploymentTolerations, job.Spec.Template.Spec.Tolerations) + } }) } } @@ -1169,9 +1259,10 @@ func TestGenerateJob_EnvVars(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { r := GitJobReconciler{ - Client: fake.NewFakeClient(), - Image: "test", - Clock: RealClock{}, + Client: getTestClient(testClientSimple, []corev1.Toleration{}), + Image: "test", + Clock: RealClock{}, + SystemNamespace: config.DefaultNamespace, } for k, v := range test.osEnv { err := os.Setenv(k, v) @@ -1189,6 +1280,7 @@ func TestGenerateJob_EnvVars(t *testing.T) { if !cmp.Equal(job.Spec.Template.Spec.InitContainers[0].Env, test.expectedInitContainerEnvVars) { t.Errorf("unexpected envVars. expected %v, but got %v", test.expectedInitContainerEnvVars, job.Spec.Template.Spec.InitContainers[0].Env) } + for k := range test.osEnv { err := os.Unsetenv(k) if err != nil { @@ -1289,9 +1381,10 @@ func TestCheckforPollingTask(t *testing.T) { } } -func httpSecretMock() client.Client { +func httpSecretMock(fleetControllerDeployment *appsv1.Deployment) client.Client { scheme := runtime.NewScheme() utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(appsv1.AddToScheme(scheme)) return fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(&corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "secretName"}, @@ -1300,12 +1393,13 @@ func httpSecretMock() client.Client { corev1.BasicAuthPasswordKey: []byte("pass"), }, Type: corev1.SecretTypeBasicAuth, - }).Build() + }, fleetControllerDeployment).Build() } -func sshSecretMock() client.Client { +func sshSecretMock(fleetControllerDeployment *appsv1.Deployment) client.Client { scheme := runtime.NewScheme() utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(appsv1.AddToScheme(scheme)) return fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(&corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "secretName"}, @@ -1313,5 +1407,41 @@ func sshSecretMock() client.Client { corev1.SSHAuthPrivateKey: []byte("ssh key"), }, Type: corev1.SecretTypeSSHAuth, - }).Build() + }, fleetControllerDeployment).Build() +} + +func defaultMockClient(fleetControllerDeployment *appsv1.Deployment) client.Client { + scheme := runtime.NewScheme() + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(appsv1.AddToScheme(scheme)) + + return fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects( + fleetControllerDeployment, + ).Build() +} + +func getFleetControllerDeployment(tolerations []corev1.Toleration) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.ManagerConfigName, + Namespace: config.DefaultNamespace, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Tolerations: tolerations, + }, + }, + }, + } +} + +func getTestClient(t testClientType, tolerations []corev1.Toleration) client.Client { + switch t { + case testClientHTTP: + return httpSecretMock(getFleetControllerDeployment(tolerations)) + case testClientSSH: + return sshSecretMock(getFleetControllerDeployment(tolerations)) + } + return defaultMockClient(getFleetControllerDeployment(tolerations)) }