fix(telemetry): Application Insights Shutdown on Retina Operator

operator/cmd/root.go

@@ -8,6 +8,7 @@ import (
 	"os"
 
 	"github.com/microsoft/retina/operator/cmd/standard"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -25,10 +26,14 @@ var (
 		Use:   "retina-operator",
 		Short: "Retina Operator",
 		Long:  "Start Retina Operator",
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(_ *cobra.Command, _ []string) error {
 			fmt.Println("Starting Retina Operator")
 			d := standard.NewOperator(metricsAddr, probeAddr, cfgFile, enableLeaderElection)
-			d.Start()
+
+			if err := d.Start(); err != nil {
+				return errors.Wrap(err, "failed to start retina-operator")
+			}
+			return nil
 		},
 	}
 )

operator/cmd/standard/deployment.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/pprof"
-	"os"
 
 	"go.uber.org/zap/zapcore"
 
@@ -39,6 +38,7 @@ import (
 	retinaendpointcontroller "github.com/microsoft/retina/pkg/controllers/operator/retinaendpoint"
 	"github.com/microsoft/retina/pkg/log"
 	"github.com/microsoft/retina/pkg/telemetry"
+	"github.com/pkg/errors"
 )
 
 var (
@@ -78,7 +78,7 @@ func NewOperator(metricsAddr, probeAddr, configFile string, enableLeaderElection
 	}
 }
 
-func (o *Operator) Start() {
+func (o *Operator) Start() error {
 	mainLogger = log.Logger().Named("main")
 
 	mainLogger.Sugar().Infof("Starting standard operator")
@@ -90,8 +90,7 @@ func (o *Operator) Start() {
 	var err error
 	oconfig, err = config.GetConfig(o.configFile)
 	if err != nil {
-		fmt.Printf("failed to load config with err %s", err.Error())
-		os.Exit(1)
+		return errors.Wrap(err, "failed to load config")
 	}
 
 	mainLogger.Sugar().Infof("Operator configuration", zap.Any("configuration", oconfig))
@@ -100,15 +99,9 @@ func (o *Operator) Start() {
 	oconfig.CaptureConfig.CaptureImageVersion = buildinfo.Version
 	oconfig.CaptureConfig.CaptureImageVersionSource = captureUtils.VersionSourceOperatorImageVersion
 
-	if err != nil {
-		fmt.Printf("failed to load config with err %s", err.Error())
-		os.Exit(1)
-	}
-
 	err = initLogging(oconfig, buildinfo.ApplicationInsightsID)
 	if err != nil {
-		fmt.Printf("failed to initialize logging with err %s", err.Error())
-		os.Exit(1)
+		return errors.Wrap(err, "failed to initialize logging")
 	}
 
 	ctrl.SetLogger(crzap.New(crzap.UseFlagOptions(opts), crzap.Encoder(zapcore.NewConsoleEncoder(log.EncoderConfig()))))
@@ -135,15 +128,13 @@ func (o *Operator) Start() {
 		// LeaderElectionReleaseOnCancel: true,
 	})
 	if err != nil {
-		mainLogger.Error("Unable to start manager", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "unable to start manager")
 	}
 
 	ctx := context.Background()
 	clientset, err := apiextv1.NewForConfig(mgr.GetConfig())
 	if err != nil {
-		mainLogger.Error("Failed to get apiextension clientset", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "failed to get apiextension clientset")
 	}
 
 	if oconfig.InstallCRDs {
@@ -152,8 +143,7 @@ func (o *Operator) Start() {
 		var crds map[string]*v1.CustomResourceDefinition
 		crds, err = deploy.InstallOrUpdateCRDs(ctx, oconfig.EnableRetinaEndpoint, clientset)
 		if err != nil {
-			mainLogger.Error("unable to register CRDs", zap.Error(err))
-			os.Exit(1)
+			return errors.Wrap(err, "unable to register CRDs")
 		}
 		for name := range crds {
 			mainLogger.Info("CRD registered", zap.String("name", name))
@@ -162,8 +152,7 @@ func (o *Operator) Start() {
 
 	apiserverURL, err := telemetry.GetK8SApiserverURLFromKubeConfig()
 	if err != nil {
-		mainLogger.Error("Apiserver URL is cannot be found", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "apiserver URL cannot be found")
 	}
 
 	var tel telemetry.Telemetry
@@ -173,10 +162,14 @@ func (o *Operator) Start() {
 			"version":                   buildinfo.Version,
 			telemetry.PropertyApiserver: apiserverURL,
 		}
+
+		telemetry.InitAppInsights(buildinfo.ApplicationInsightsID, buildinfo.Version)
+		defer telemetry.ShutdownAppInsights()
+		defer telemetry.TrackPanic()
+
 		tel, err = telemetry.NewAppInsightsTelemetryClient("retina-operator", properties)
 		if err != nil {
-			mainLogger.Error("failed to create telemetry client", zap.Error(err))
-			os.Exit(1)
+			return errors.Wrap(err, "failed to create telemetry client")
 		}
 	} else {
 		mainLogger.Info("telemetry disabled", zap.String("apiserver", apiserverURL))
@@ -185,20 +178,17 @@ func (o *Operator) Start() {
 
 	kubeClient, err := kubernetes.NewForConfig(mgr.GetConfig())
 	if err != nil {
-		mainLogger.Error("Failed to get clientset", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "failed to get clientset")
 	}
 
 	captureReconciler, err := captureController.NewCaptureReconciler(
 		mgr.GetClient(), mgr.GetScheme(), kubeClient, oconfig.CaptureConfig,
 	)
 	if err != nil {
-		mainLogger.Error("Unable to create capture reconciler", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "unable to create capture reconciler")
 	}
 	if err = captureReconciler.SetupWithManager(mgr); err != nil {
-		mainLogger.Error("Unable to setup retina capture controller with manager", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "unable to setup retina capture controller with manager")
 	}
 
 	ctrlCtx := ctrl.SetupSignalHandler()
@@ -222,37 +212,34 @@ func (o *Operator) Start() {
 
 			pc := podcontroller.New(mgr.GetClient(), mgr.GetScheme(), retinaendpointchannel)
 			if err = (pc).SetupWithManager(mgr); err != nil {
-				mainLogger.Error("Unable to create controller", zap.String("controller", "podcontroller"), zap.Error(err))
-				os.Exit(1)
+				return errors.Wrap(err, "unable to create controller - podcontroller")
 			}
 		}
 	}
 
 	mc := metricsconfiguration.New(mgr.GetClient(), mgr.GetScheme())
 	if err = (mc).SetupWithManager(mgr); err != nil {
-		mainLogger.Error("Unable to create controller", zap.String("controller", "metricsconfiguration"), zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "unable to create controller - metricsconfiguration")
 	}
 
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
-		mainLogger.Error("Unable to set up health check", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "unable to set up health check")
 	}
 	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
-		mainLogger.Error("Unable to set up ready check", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "unable to set up ready check")
 	}
 
+	// start heartbeat goroutine for application insights
+	go tel.Heartbeat(ctx, oconfig.TelemetryInterval)
+
 	mainLogger.Info("Starting manager")
 	if err := mgr.Start(ctrlCtx); err != nil {
-		mainLogger.Error("Problem running manager", zap.Error(err))
-		os.Exit(1)
+		return errors.Wrap(err, "problem running manager")
 	}
 
-	// start heartbeat goroutine for application insights
-	go tel.Heartbeat(ctx, oconfig.TelemetryInterval)
+	return nil
 }
 
 func EnablePProf() {