feat: add manual layer sync (#321)

Author: LucasMrqes

internal/annotations/annotations.go

@@ -24,6 +24,8 @@ const (
 
 	ForceApply              string = "notifications.terraform.padok.cloud/force-apply"
 	AdditionnalTriggerPaths string = "config.terraform.padok.cloud/additionnal-trigger-paths"
+
+	SyncNow string = "api.terraform.padok.cloud/sync-now"
 )
 
 func Add(ctx context.Context, c client.Client, obj client.Object, annotations map[string]string) error {

internal/controllers/terraformlayer/conditions.go

@@ -9,6 +9,7 @@ import (
 
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
 	"github.com/padok-team/burrito/internal/annotations"
+	log "github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -243,6 +244,31 @@ func (r *Reconciler) IsApplyUpToDate(t *configv1alpha1.TerraformLayer) (metav1.C
 	return condition, true
 }
 
+func (r *Reconciler) IsSyncScheduled(t *configv1alpha1.TerraformLayer) (metav1.Condition, bool) {
+	condition := metav1.Condition{
+		Type:               "IsSyncScheduled",
+		ObservedGeneration: t.GetObjectMeta().GetGeneration(),
+		Status:             metav1.ConditionUnknown,
+		LastTransitionTime: metav1.NewTime(time.Now()),
+	}
+	// check if annotations.SyncNow is present
+	if _, ok := t.Annotations[annotations.SyncNow]; ok {
+		condition.Reason = "SyncScheduled"
+		condition.Message = "A sync has been manually scheduled"
+		condition.Status = metav1.ConditionTrue
+		// Remove the annotation to avoid running the sync again
+		err := annotations.Remove(context.Background(), r.Client, t, annotations.SyncNow)
+		if err != nil {
+			log.Errorf("Failed to remove annotation %s from layer %s: %s", annotations.SyncNow, t.Name, err)
+		}
+		return condition, true
+	}
+	condition.Reason = "NoSyncScheduled"
+	condition.Message = "No sync has been manually scheduled"
+	condition.Status = metav1.ConditionFalse
+	return condition, false
+}
+
 func LayerFilesHaveChanged(layer configv1alpha1.TerraformLayer, changedFiles []string) bool {
 	if len(changedFiles) == 0 {
 		return true

internal/controllers/terraformlayer/controller_test.go

@@ -313,6 +313,33 @@ var _ = Describe("Layer", func() {
 				Expect(result.RequeueAfter).To(Equal(reconciler.Config.Controller.Timers.DriftDetection))
 			})
 		})
+		Describe("When a TerraformLayer is annotated to be manually synced", Ordered, func() {
+			BeforeAll(func() {
+				name = types.NamespacedName{
+					Name:      "nominal-case-8",
+					Namespace: "default",
+				}
+				result, layer, reconcileError, err = getResult(name)
+			})
+			It("should still exists", func() {
+				Expect(err).NotTo(HaveOccurred())
+			})
+			It("should not return an error", func() {
+				Expect(reconcileError).NotTo(HaveOccurred())
+			})
+			It("should end in PlanNeeded state", func() {
+				Expect(layer.Status.State).To(Equal("PlanNeeded"))
+			})
+			It("should set RequeueAfter to WaitAction", func() {
+				Expect(result.RequeueAfter).To(Equal(reconciler.Config.Controller.Timers.WaitAction))
+			})
+			It("should have created a plan TerraformRun", func() {
+				runs, err := getLinkedRuns(k8sClient, layer)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(runs.Items)).To(Equal(1))
+				Expect(runs.Items[0].Spec.Action).To(Equal("plan"))
+			})
+		})
 	})
 	Describe("When a TerraformLayer has errored on plan and is still before new DriftDetection tick", Ordered, func() {
 		BeforeAll(func() {

internal/controllers/terraformlayer/states.go

@@ -25,14 +25,18 @@ func (r *Reconciler) GetState(ctx context.Context, layer *configv1alpha1.Terrafo
 	c3, IsLastRelevantCommitPlanned := r.IsLastRelevantCommitPlanned(layer)
 	c4, HasLastPlanFailed := r.HasLastPlanFailed(layer)
 	c5, IsApplyUpToDate := r.IsApplyUpToDate(layer)
-	conditions := []metav1.Condition{c1, c2, c3, c4, c5}
+	c6, IsSyncScheduled := r.IsSyncScheduled(layer)
+	conditions := []metav1.Condition{c1, c2, c3, c4, c5, c6}
 	switch {
 	case IsRunning:
 		log.Infof("layer %s is running, waiting for the run to finish", layer.Name)
 		return &Idle{}, conditions
 	case IsLastPlanTooOld || !IsLastRelevantCommitPlanned:
 		log.Infof("layer %s has an outdated plan, creating a new run", layer.Name)
 		return &PlanNeeded{}, conditions
+	case IsSyncScheduled:
+		log.Infof("layer %s has a sync scheduled, creating a new run", layer.Name)
+		return &PlanNeeded{}, conditions
 	case !IsApplyUpToDate && !HasLastPlanFailed:
 		log.Infof("layer %s needs to be applied, creating a new run", layer.Name)
 		return &ApplyNeeded{}, conditions

internal/controllers/terraformlayer/testdata/nominal-case.yaml

@@ -201,3 +201,26 @@ status:
   lastRun:
     name: run-running
     namespace: default
+---
+apiVersion: config.terraform.padok.cloud/v1alpha1
+kind: TerraformLayer
+metadata:
+  labels:
+    app.kubernetes.io/instance: in-cluster-burrito
+  annotations:
+    api.terraform.padok.cloud/sync-now: "true"
+  name: nominal-case-8
+  namespace: default
+spec:
+  branch: main
+  path: nominal-case-eight/
+  remediationStrategy:
+    autoApply: true
+  repository:
+    name: burrito
+    namespace: default
+  terraform:
+    terragrunt:
+      enabled: true
+      version: 0.45.4
+    version: 1.3.1

internal/server/api/layers.go

@@ -9,24 +9,26 @@ import (
 	"github.com/labstack/echo/v4"
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
 	"github.com/padok-team/burrito/internal/annotations"
+	"github.com/padok-team/burrito/internal/server/utils"
 	log "github.com/sirupsen/logrus"
 )
 
 type layer struct {
-	UID        string `json:"uid"`
-	Name       string `json:"name"`
-	Namespace  string `json:"namespace"`
-	Repository string `json:"repository"`
-	Branch     string `json:"branch"`
-	Path       string `json:"path"`
-	State      string `json:"state"`
-	RunCount   int    `json:"runCount"`
-	LastRun    Run    `json:"lastRun"`
-	LastRunAt  string `json:"lastRunAt"`
-	LastResult string `json:"lastResult"`
-	IsRunning  bool   `json:"isRunning"`
-	IsPR       bool   `json:"isPR"`
-	LatestRuns []Run  `json:"latestRuns"`
+	UID              string                 `json:"uid"`
+	Name             string                 `json:"name"`
+	Namespace        string                 `json:"namespace"`
+	Repository       string                 `json:"repository"`
+	Branch           string                 `json:"branch"`
+	Path             string                 `json:"path"`
+	State            string                 `json:"state"`
+	RunCount         int                    `json:"runCount"`
+	LastRun          Run                    `json:"lastRun"`
+	LastRunAt        string                 `json:"lastRunAt"`
+	LastResult       string                 `json:"lastResult"`
+	IsRunning        bool                   `json:"isRunning"`
+	IsPR             bool                   `json:"isPR"`
+	LatestRuns       []Run                  `json:"latestRuns"`
+	ManualSyncStatus utils.ManualSyncStatus `json:"manualSyncStatus"`
 }
 
 type Run struct {
@@ -83,20 +85,21 @@ func (a *API) LayersHandler(c echo.Context) error {
 			running = runStillRunning(run)
 		}
 		results = append(results, layer{
-			UID:        string(l.UID),
-			Name:       l.Name,
-			Namespace:  l.Namespace,
-			Repository: fmt.Sprintf("%s/%s", l.Spec.Repository.Namespace, l.Spec.Repository.Name),
-			Branch:     l.Spec.Branch,
-			Path:       l.Spec.Path,
-			State:      a.getLayerState(l),
-			RunCount:   len(l.Status.LatestRuns),
-			LastRun:    runAPI,
-			LastRunAt:  l.Status.LastRun.Date.Format(time.RFC3339),
-			LastResult: l.Status.LastResult,
-			IsRunning:  running,
-			IsPR:       a.isLayerPR(l),
-			LatestRuns: transformLatestRuns(l.Status.LatestRuns),
+			UID:              string(l.UID),
+			Name:             l.Name,
+			Namespace:        l.Namespace,
+			Repository:       fmt.Sprintf("%s/%s", l.Spec.Repository.Namespace, l.Spec.Repository.Name),
+			Branch:           l.Spec.Branch,
+			Path:             l.Spec.Path,
+			State:            a.getLayerState(l),
+			RunCount:         len(l.Status.LatestRuns),
+			LastRun:          runAPI,
+			LastRunAt:        l.Status.LastRun.Date.Format(time.RFC3339),
+			LastResult:       l.Status.LastResult,
+			IsRunning:        running,
+			IsPR:             a.isLayerPR(l),
+			LatestRuns:       transformLatestRuns(l.Status.LatestRuns),
+			ManualSyncStatus: utils.GetManualSyncStatus(l),
 		})
 	}
 	return c.JSON(http.StatusOK, &layersResponse{

internal/server/api/sync.go

@@ -0,0 +1,38 @@
+package api
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/labstack/echo/v4"
+	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
+	"github.com/padok-team/burrito/internal/annotations"
+	"github.com/padok-team/burrito/internal/server/utils"
+	log "github.com/sirupsen/logrus"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func (a *API) SyncLayerHandler(c echo.Context) error {
+	layer := &configv1alpha1.TerraformLayer{}
+	err := a.Client.Get(context.Background(), client.ObjectKey{
+		Namespace: c.Param("namespace"),
+		Name:      c.Param("layer"),
+	}, layer)
+	if err != nil {
+		log.Errorf("could not get terraform layer: %s", err)
+		return c.JSON(http.StatusInternalServerError, map[string]string{"error": "An error occurred while getting the layer"})
+	}
+	syncStatus := utils.GetManualSyncStatus(*layer)
+	if syncStatus == utils.ManualSyncAnnotated || syncStatus == utils.ManualSyncPending {
+		return c.JSON(http.StatusConflict, map[string]string{"error": "Layer sync already triggered"})
+	}
+
+	err = annotations.Add(context.Background(), a.Client, layer, map[string]string{
+		annotations.SyncNow: "true",
+	})
+	if err != nil {
+		log.Errorf("could not update terraform layer annotations: %s", err)
+		return c.JSON(http.StatusInternalServerError, map[string]string{"error": "An error occurred while updating the layer annotations"})
+	}
+	return c.JSON(http.StatusOK, map[string]string{"status": "Layer sync triggered"})
+}

internal/server/server.go

@@ -82,6 +82,7 @@ func (s *Server) Exec() {
 	e.GET("/healthz", handleHealthz)
 	api.POST("/webhook", s.Webhook.GetHttpHandler())
 	api.GET("/layers", s.API.LayersHandler)
+	api.POST("/layers/:namespace/:layer/sync", s.API.SyncLayerHandler)
 	api.GET("/repositories", s.API.RepositoriesHandler)
 	api.GET("/logs/:namespace/:layer/:run/:attempt", s.API.GetLogsHandler)
 	api.GET("/run/:namespace/:layer/:run/attempts", s.API.GetAttemptsHandler)

internal/server/utils/manual_sync.go

@@ -0,0 +1,27 @@
+package utils
+
+import (
+	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
+	"github.com/padok-team/burrito/internal/annotations"
+)
+
+type ManualSyncStatus string
+
+const (
+	ManualSyncNone      ManualSyncStatus = "none"
+	ManualSyncAnnotated ManualSyncStatus = "annotated"
+	ManualSyncPending   ManualSyncStatus = "pending"
+)
+
+func GetManualSyncStatus(layer configv1alpha1.TerraformLayer) ManualSyncStatus {
+	if layer.Annotations[annotations.SyncNow] == "true" {
+		return ManualSyncAnnotated
+	}
+	// check the IsSyncScheduled condition on layer
+	for _, c := range layer.Status.Conditions {
+		if c.Type == "IsSyncScheduled" && c.Status == "True" {
+			return ManualSyncPending
+		}
+	}
+	return ManualSyncNone
+}

ui/package.json

@@ -16,6 +16,7 @@
     "axios": "^1.5.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-focus-lock": "^2.13.2",
     "react-router-dom": "^6.16.0",
     "react-tooltip": "^5.21.6",
     "tailwind-merge": "^2.0.0"