1409

Signed-off-by: 刘硕 <[email protected]>
openkruise · Sep 20, 2023 · b8bb6cc · b8bb6cc
1 parent 12e6c38
commit b8bb6cc
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 9 deletions.
diff --git a/apis/apps/pub/launch_priority.go b/apis/apps/pub/launch_priority.go
@@ -20,6 +20,10 @@ const (
 	// ContainerLaunchPriorityEnvName is the env name that users have to define in pod container
 	// to identity the launch priority of this container.
 	ContainerLaunchPriorityEnvName = "KRUISE_CONTAINER_PRIORITY"
+	// ContainerLaunchTimeOutEnvName is high priority container startup times out.
+	ContainerLaunchTimeOutEnvName = "KRUISE_CONTAINER_LAUNCH_TIMEOUT"
+	// ContainerLaunchPriorityUpdateTimeKey a label used to record the update time.
+	ContainerLaunchPriorityUpdateTimeKey = "apps.kruise.io/container-launch-priority-update-time"
 	// ContainerLaunchBarrierEnvName is the env name that Kruise webhook will inject into containers
 	// if the pod have configured launch priority.
 	ContainerLaunchBarrierEnvName = "KRUISE_CONTAINER_BARRIER"

diff --git a/pkg/controller/containerlaunchpriority/container_launch_priority_controller.go b/pkg/controller/containerlaunchpriority/container_launch_priority_controller.go
@@ -19,9 +19,13 @@ package containerlauchpriority
 import (
 	"context"
 	"fmt"
+	"os"
 	"strconv"
 	"time"
 
+	"github.com/google/uuid"
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
+
 	"github.com/openkruise/kruise/pkg/util"
 	utilclient "github.com/openkruise/kruise/pkg/util/client"
 	utilcontainerlaunchpriority "github.com/openkruise/kruise/pkg/util/containerlaunchpriority"
@@ -132,6 +136,9 @@ func (r *ReconcileContainerLaunchPriority) Reconcile(_ context.Context, request
 	}
 	err = r.Get(context.TODO(), barrierNamespacedName, barrier)
 	if errors.IsNotFound(err) {
+		barrier.Annotations = map[string]string{
+			appspub.ContainerLaunchPriorityUpdateTimeKey: time.Now().Format(time.RFC3339),
+		}
 		barrier.Namespace = pod.GetNamespace()
 		barrier.Name = pod.Name + "-barrier"
 		barrier.OwnerReferences = append(barrier.OwnerReferences, metav1.OwnerReference{
@@ -151,23 +158,76 @@ func (r *ReconcileContainerLaunchPriority) Reconcile(_ context.Context, request
 		return reconcile.Result{}, err
 	}
 
+	var requeueTime time.Duration
 	// set next starting containers
 	_, containersReady := podutil.GetPodCondition(&pod.Status, v1.ContainersReady)
 	if containersReady != nil && containersReady.Status != v1.ConditionTrue {
-		patchKey := r.findNextPatchKey(pod)
+		patchKey, timeout, container := r.findNextPatchKey(pod)
 		if patchKey == nil {
 			return reconcile.Result{}, nil
 		}
-		key := "p_" + strconv.Itoa(*patchKey)
-		if err = r.patchOnKeyNotExist(barrier, key); err != nil {
-			return reconcile.Result{}, err
+		updateTime := time.Now()
+		if barrier.Labels != nil {
+			updateStr := barrier.Labels[appspub.ContainerLaunchPriorityUpdateTimeKey]
+			parse, err := time.Parse(time.RFC3339, updateStr)
+			if err != nil {
+				updateTime = parse
+			}
 		}
+		containerStatus := util.GetContainerStatus(container.Name, pod)
+
+		if time.Duration(timeout)*time.Second < time.Since(updateTime) && (containerStatus == nil || containerStatus.Ready == false) {
+			name, _ := os.Hostname()
+			event := &v1.Event{
+				TypeMeta: metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      barrier.Name + uuid.New().String(),
+					Namespace: barrier.Namespace,
+				},
+				InvolvedObject: v1.ObjectReference{
+					Kind:            barrier.Kind,
+					Namespace:       barrier.Namespace,
+					Name:            barrier.Name,
+					UID:             barrier.UID,
+					APIVersion:      barrier.APIVersion,
+					ResourceVersion: barrier.ResourceVersion,
+				},
+				ReportingController: "kruise-manager",
+				ReportingInstance:   name,
+				Action:              "Update",
+				Reason:              "ContainerLaunchTimeout",
+				Message:             fmt.Sprintf("Container %s has not launched successfully more than %ss.", container.Name, strconv.Itoa(timeout)),
+				Source: v1.EventSource{
+					Component: "kruise-manager",
+					Host:      name,
+				},
+				FirstTimestamp: metav1.Now(),
+				LastTimestamp:  metav1.Now(),
+				Count:          1,
+				Type:           "Warning",
+				EventTime:      metav1.NewMicroTime(time.Now()),
+			}
+			err := r.Client.Create(context.Background(), event)
+			if err != nil {
+				klog.Errorf("failed to create event %s, error: %v", event.Name, err)
+			}
+		}
+
+		if time.Duration(timeout)*time.Second-time.Since(updateTime) > 0 {
+			requeueTime = time.Duration(timeout)*time.Second - time.Since(updateTime)
+		} else {
+			key := "p_" + strconv.Itoa(*patchKey)
+			if err = r.patchOnKeyNotExist(barrier, key); err != nil {
+				return reconcile.Result{}, err
+			}
+		}
+
 	}
 
-	return reconcile.Result{}, nil
+	return reconcile.Result{RequeueAfter: requeueTime}, nil
 }
 
-func (r *ReconcileContainerLaunchPriority) findNextPatchKey(pod *v1.Pod) *int {
+func (r *ReconcileContainerLaunchPriority) findNextPatchKey(pod *v1.Pod) (*int, int, *v1.Container) {
 	var priority *int
 	var containerPendingSet = make(map[string]bool)
 	for _, status := range pod.Status.ContainerStatuses {
@@ -176,6 +236,21 @@ func (r *ReconcileContainerLaunchPriority) findNextPatchKey(pod *v1.Pod) *int {
 		}
 		containerPendingSet[status.Name] = true
 	}
+
+	getTimeout := func(c v1.Container) int {
+		for _, e := range c.Env {
+			if e.Name == appspub.ContainerLaunchTimeOutEnvName {
+				p, _ := strconv.Atoi(e.Value)
+				if p < 0 {
+					return 0
+				}
+				return p
+			}
+		}
+		return 0
+	}
+	timeout := 0
+	var container *v1.Container
 	for _, c := range pod.Spec.Containers {
 		if _, ok := containerPendingSet[c.Name]; ok {
 			p := utilcontainerlaunchpriority.GetContainerPriority(&c)
@@ -184,17 +259,20 @@ func (r *ReconcileContainerLaunchPriority) findNextPatchKey(pod *v1.Pod) *int {
 			}
 			if priority == nil || *p > *priority {
 				priority = p
+				timeout = getTimeout(c)
+				container = &c
+
 			}
 		}
 	}
-	return priority
+	return priority, timeout, container
 }
 
 func (r *ReconcileContainerLaunchPriority) patchOnKeyNotExist(barrier *v1.ConfigMap, key string) error {
 	if _, ok := barrier.Data[key]; !ok {
 		body := fmt.Sprintf(
-			`{"data":{"%s":"true"}}`,
-			key,
+			`{"spec":{"data":{"%s":"true"}},"metadata":{"annotations":{"%s":"%s"}}}`,
+			key, appspub.ContainerLaunchPriorityUpdateTimeKey, time.Now().Format(time.RFC3339),
 		)
 		return r.Client.Patch(context.TODO(), barrier, client.RawPatch(types.StrategicMergePatchType, []byte(body)))
 	}