Hide controller logs in integrationtests (rancher#3326)

This introduces new environment variables for integrationtests:

* CI_SILENCE_CTRL - no logging output, even when test fails
* CI_KUBECONFIG - write kubeconfig for testenv to this path
* CI_USE_EXISTING_CLUSTER - integrationtests don't setup testenv, but
  use existing cluster from current KUBECONFIG

integrationtests/agent/suite_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -30,7 +29,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
@@ -73,15 +71,10 @@ var _ = BeforeSuite(func() {
 	SetDefaultEventuallyTimeout(timeout)
 
 	ctx, cancel = context.WithCancel(context.TODO())
-	existing := os.Getenv("CI_USE_EXISTING_CLUSTER") == "true"
-	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "charts", "fleet-crd", "templates", "crds.yaml")},
-		ErrorIfCRDPathMissing: true,
-		UseExistingCluster:    &existing,
-	}
+	testEnv = utils.NewEnvTest("../..")
 
 	var err error
-	cfg, err = testEnv.Start()
+	cfg, err = utils.StartTestEnv(testEnv)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 
@@ -92,11 +85,6 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())
 
-	zopts := zap.Options{
-		Development: true,
-	}
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zopts)))
-
 	k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:         scheme.Scheme,
 		LeaderElection: false,

integrationtests/cli/cleanup/suite_test.go

@@ -2,8 +2,6 @@ package cleanup
 
 import (
 	"context"
-	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -20,7 +18,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -49,18 +46,12 @@ func TestFleetCLICleanUp(t *testing.T) {
 var _ = BeforeSuite(func() {
 	SetDefaultEventuallyTimeout(timeout)
 	ctx, cancel = context.WithCancel(context.TODO())
-	if os.Getenv("DEBUG") != "" {
-		ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true})))
-	}
-	ctx = log.IntoContext(ctx, ctrl.Log)
 
-	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "fleet-crd", "templates", "crds.yaml")},
-		ErrorIfCRDPathMissing: true,
-	}
+	testEnv = utils.NewEnvTest("../../..")
+	ctx = log.IntoContext(ctx, ctrl.Log)
 
 	var err error
-	cfg, err = testEnv.Start()
+	cfg, err = utils.StartTestEnv(testEnv)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 

integrationtests/cli/deploy/suite_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"os"
 	"path"
-	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -42,13 +41,10 @@ func TestFleetDeploy(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	ctx, cancel = context.WithCancel(context.TODO())
-	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "fleet-crd", "templates", "crds.yaml")},
-		ErrorIfCRDPathMissing: true,
-	}
+	testEnv = utils.NewEnvTest("../../..")
 
 	var err error
-	cfg, err = testEnv.Start()
+	cfg, err = utils.StartTestEnv(testEnv)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 

integrationtests/cli/target/suite_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"os"
 	"path"
-	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -42,13 +41,10 @@ func TestFleetTarget(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	ctx, cancel = context.WithCancel(context.TODO())
-	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "fleet-crd", "templates", "crds.yaml")},
-		ErrorIfCRDPathMissing: true,
-	}
+	testEnv = utils.NewEnvTest("../../..")
 
 	var err error
-	cfg, err = testEnv.Start()
+	cfg, err = utils.StartTestEnv(testEnv)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 

integrationtests/controller/bundle/suite_test.go

@@ -18,19 +18,16 @@ import (
 
 	"k8s.io/client-go/rest"
 
-	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 var (
-	cancel      context.CancelFunc
-	cfg         *rest.Config
-	ctx         context.Context
-	k8sClient   client.Client
-	testenv     *envtest.Environment
-	testEnvBool bool
+	cancel    context.CancelFunc
+	cfg       *rest.Config
+	ctx       context.Context
+	k8sClient client.Client
+	testenv   *envtest.Environment
 
 	namespace string
 )
@@ -42,18 +39,15 @@ func TestFleet(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	ctx, cancel = context.WithCancel(context.TODO())
-	testenv = utils.NewEnvTest()
-	testEnvBool = true
+	testenv = utils.NewEnvTest("../../..")
 
 	var err error
-	cfg, err = testenv.Start()
+	cfg, err = utils.StartTestEnv(testenv)
 	Expect(err).NotTo(HaveOccurred())
 
 	k8sClient, err = utils.NewClient(cfg)
 	Expect(err).NotTo(HaveOccurred())
 
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true})))
-
 	mgr, err := utils.NewManager(cfg)
 	Expect(err).ToNot(HaveOccurred())
 

integrationtests/controller/bundledeployment/suite_test.go

@@ -13,10 +13,8 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/rest"
-	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 var (
@@ -36,17 +34,15 @@ func TestFleet(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	ctx, cancel = context.WithCancel(context.TODO())
-	testenv = utils.NewEnvTest()
+	testenv = utils.NewEnvTest("../../..")
 
 	var err error
-	cfg, err = testenv.Start()
+	cfg, err = utils.StartTestEnv(testenv)
 	Expect(err).NotTo(HaveOccurred())
 
 	k8sClient, err = utils.NewClient(cfg)
 	Expect(err).NotTo(HaveOccurred())
 
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true})))
-
 	mgr, err := utils.NewManager(cfg)
 	Expect(err).ToNot(HaveOccurred())
 

integrationtests/controller/cluster/suite_test.go

@@ -16,10 +16,8 @@ import (
 
 	"k8s.io/client-go/rest"
 
-	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 var (
@@ -39,17 +37,15 @@ func TestFleet(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	ctx, cancel = context.WithCancel(context.TODO())
-	testenv = utils.NewEnvTest()
+	testenv = utils.NewEnvTest("../../..")
 
 	var err error
-	cfg, err = testenv.Start()
+	cfg, err = utils.StartTestEnv(testenv)
 	Expect(err).NotTo(HaveOccurred())
 
 	k8sClient, err = utils.NewClient(cfg)
 	Expect(err).NotTo(HaveOccurred())
 
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true})))
-
 	mgr, err := utils.NewManager(cfg)
 	Expect(err).ToNot(HaveOccurred())
 

integrationtests/controller/clustergroup/suite_test.go

@@ -13,10 +13,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"k8s.io/client-go/rest"
-	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 var (
@@ -36,17 +34,15 @@ func TestFleet(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	ctx, cancel = context.WithCancel(context.TODO())
-	testenv = utils.NewEnvTest()
+	testenv = utils.NewEnvTest("../../..")
 
 	var err error
-	cfg, err = testenv.Start()
+	cfg, err = utils.StartTestEnv(testenv)
 	Expect(err).NotTo(HaveOccurred())
 
 	k8sClient, err = utils.NewClient(cfg)
 	Expect(err).NotTo(HaveOccurred())
 
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true})))
-
 	mgr, err := utils.NewManager(cfg)
 	Expect(err).ToNot(HaveOccurred())
 

integrationtests/helmops/controller/suite_test.go

@@ -1,16 +1,15 @@
 package controller
 
 import (
-	"bytes"
 	"context"
-	"path/filepath"
 	"testing"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"go.uber.org/mock/gomock"
 
+	"github.com/rancher/fleet/integrationtests/utils"
 	"github.com/rancher/fleet/internal/cmd/controller/helmops/reconciler"
 	ctrlreconciler "github.com/rancher/fleet/internal/cmd/controller/reconciler"
 	"github.com/rancher/fleet/internal/cmd/controller/target"
@@ -24,7 +23,6 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 const (
@@ -37,7 +35,6 @@ var (
 	ctx          context.Context
 	cancel       context.CancelFunc
 	k8sClient    client.Client
-	logsBuffer   bytes.Buffer
 	namespace    string
 	k8sClientSet *kubernetes.Clientset
 )
@@ -50,13 +47,10 @@ func TestGitJobController(t *testing.T) {
 var _ = BeforeSuite(func() {
 	SetDefaultEventuallyTimeout(timeout)
 	ctx, cancel = context.WithCancel(context.TODO())
-	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "fleet-crd", "templates", "crds.yaml")},
-		ErrorIfCRDPathMissing: true,
-	}
+	testEnv = utils.NewEnvTest("../../..")
 
 	var err error
-	cfg, err = testEnv.Start()
+	cfg, err = utils.StartTestEnv(testEnv)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 
@@ -77,10 +71,6 @@ var _ = BeforeSuite(func() {
 
 	ctlr := gomock.NewController(GinkgoT())
 
-	// redirect logs to a buffer that we can read in the tests
-	GinkgoWriter.TeeTo(&logsBuffer)
-	ctrl.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
-
 	config.Set(&config.Config{})
 
 	err = (&reconciler.HelmAppReconciler{

integrationtests/utils/envtest.go

@@ -1,9 +1,12 @@
 package utils
 
 import (
+	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/go-logr/logr"
+	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 
 	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
@@ -14,6 +17,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 )
 
@@ -28,11 +33,33 @@ func init() {
 }
 
 // NewEnvTest returns a new envtest with the Fleet CRDs loaded.
-func NewEnvTest() *envtest.Environment {
+// Run ginkgo with the -v flag to see the logs in real time.
+func NewEnvTest(root string) *envtest.Environment {
+	if os.Getenv("CI_SILENCE_CTRL") != "" {
+		ctrl.SetLogger(logr.New(log.NullLogSink{}))
+	} else {
+		ctrl.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true)))
+	}
+
+	existing := os.Getenv("CI_USE_EXISTING_CLUSTER") == "true"
 	return &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "fleet-crd", "templates", "crds.yaml")},
+		CRDDirectoryPaths:     []string{filepath.Join(root, "charts", "fleet-crd", "templates", "crds.yaml")},
 		ErrorIfCRDPathMissing: true,
+		UseExistingCluster:    &existing,
+	}
+}
+
+func StartTestEnv(testEnv *envtest.Environment) (*rest.Config, error) {
+	cfg, err := testEnv.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	if config := os.Getenv("CI_KUBECONFIG"); config != "" {
+		err = WriteKubeConfig(cfg, config)
 	}
+
+	return cfg, err
 }
 
 // NewClient returns a new controller-runtime client.