Shut runnables down gracefully and log so

This commit

 - fixes a problem whereby the runner loop didn't exit (the good old
   "break out of the select but not out of the for" mistake)
 - uses a WaitGroup to make sure all the runner children (i.e., client
   caches) have exited
 - puts in more logging of things starting up and shutting down

Signed-off-by: Michael Bridgen <[email protected]>

Author: squaremo

controllers/leveltriggered/caching.go

@@ -63,7 +63,7 @@ func (c *caches) setupWithManager(mgr ctrl.Manager) error {
 	c.baseLogger = mgr.GetLogger().WithValues("component", "target-cache")
 	c.reader = mgr.GetClient() // this specifically gets the client that has the indexing installed below; i.e., these are coupled.
 
-	c.runner = newRunner()
+	c.runner = newRunner(mgr.GetLogger().WithValues("component", "cache-runner"))
 	if err := mgr.Add(c.runner); err != nil {
 		return err
 	}
@@ -193,7 +193,7 @@ func (c *caches) watchTargetAndGetReader(ctx context.Context, clusterObject *clu
 				return nil, false, err
 			}
 
-			cancel := c.runner.run(func(ctx context.Context) {
+			cancel := c.runner.run("cache-"+cacheKey.String(), func(ctx context.Context) {
 				if err := ca.Start(ctx); err != nil {
 					logger.Error(err, "cache exited with error")
 				}
@@ -373,6 +373,7 @@ func (gc *gc) loop() {
 	for {
 		item, shutdown := gc.queue.Get()
 		if shutdown {
+			gc.log.Info("exiting cache GC loop")
 			return
 		}
 		key, ok := item.(clusterAndGVK)

controllers/leveltriggered/runner.go

@@ -1,6 +1,10 @@
 package leveltriggered
 
-import "context"
+import (
+	"context"
+	"github.com/go-logr/logr"
+	"sync"
+)
 
 // This is a dead simple way to run things using a manager's context as a base, so that they will
 // get shut down when the manager does. It must be constructed with `newRunner`, and added to a manager:
@@ -19,29 +23,33 @@ import "context"
 // It'll deadlock if you call `run` before adding it to a manager (or otherwise calling `Start`).
 
 type runWithContext struct {
-	ctx context.Context
-	run func(context.Context)
+	name string
+	ctx  context.Context
+	do   func(context.Context)
 }
 
 type runner struct {
+	log         logr.Logger
 	rootContext context.Context
 	tostart     chan runWithContext
 	ready       chan struct{}
 }
 
-func newRunner() *runner {
+func newRunner(log logr.Logger) *runner {
 	return &runner{
+		log:     log,
 		tostart: make(chan runWithContext),
 		ready:   make(chan struct{}),
 	}
 }
 
-func (r *runner) run(fn func(ctx context.Context)) context.CancelFunc {
+func (r *runner) run(name string, fn func(ctx context.Context)) context.CancelFunc {
 	<-r.ready // wait until there's a root context
 	ctx, cancel := context.WithCancel(r.rootContext)
 	r.tostart <- runWithContext{
-		run: fn,
-		ctx: ctx,
+		name: name,
+		do:   fn,
+		ctx:  ctx,
 	}
 	return cancel
 }
@@ -51,12 +59,24 @@ func (r *runner) run(fn func(ctx context.Context)) context.CancelFunc {
 func (r *runner) Start(ctx context.Context) error {
 	r.rootContext = ctx
 	close(r.ready) // broadcast that things can be run
+	var wg sync.WaitGroup
+loop:
 	for {
 		select {
 		case randc := <-r.tostart:
-			go randc.run(randc.ctx)
+			r.log.Info("starting child", "name", randc.name)
+			wg.Add(1)
+			go func(rc runWithContext) {
+				defer wg.Done()
+				rc.do(rc.ctx)
+				r.log.Info("child exited", "name", rc.name)
+			}(randc)
 		case <-r.rootContext.Done():
-			return nil
+			break loop
 		}
 	}
+	r.log.Info("Stopping and waiting for children")
+	wg.Wait()
+	r.log.Info("All children stopped; runner exit")
+	return nil
 }

controllers/leveltriggered/suite_test.go

@@ -176,11 +176,13 @@ func TestMain(m *testing.M) {
 
 	cancel()
 	wg.Wait()
+	log.Println("manager exited")
 
 	err = testEnv.Stop()
 	if err != nil {
-		log.Fatalf("stoping test env failed: %s", err)
+		log.Fatalf("stopping test env failed: %s", err)
 	}
+	log.Println("test env stopped")
 
 	os.Exit(retCode)
 }

controllers/suite_test.go

@@ -163,7 +163,7 @@ func TestMain(m *testing.M) {
 
 	err = testEnv.Stop()
 	if err != nil {
-		log.Fatalf("stoping test env failed: %s", err)
+		log.Fatalf("stopping test env failed: %s", err)
 	}
 
 	os.Exit(retCode)