From a36ff34d68965a13bb1cf7dca92b475754405274 Mon Sep 17 00:00:00 2001 From: Liu Zhenwei Date: Sat, 19 Mar 2022 21:32:48 +0800 Subject: [PATCH 1/4] inplaceupdate restart count Signed-off-by: Liu Zhenwei Signed-off-by: liuzhenwei --- apis/apps/pub/inplace_update.go | 16 ++++ .../cloneset/sync/cloneset_update.go | 2 - .../cloneset/sync/cloneset_update_test.go | 11 ++- pkg/util/inplaceupdate/inplace_update.go | 80 ++++++++++++++++--- pkg/util/inplaceupdate/inplace_update_test.go | 19 +++-- 5 files changed, 110 insertions(+), 18 deletions(-) diff --git a/apis/apps/pub/inplace_update.go b/apis/apps/pub/inplace_update.go index 8274473e21..768ae4bcf5 100644 --- a/apis/apps/pub/inplace_update.go +++ b/apis/apps/pub/inplace_update.go @@ -45,6 +45,12 @@ const ( // RuntimeContainerMetaKey is a key in pod annotations. Kruise-daemon should report the // states of runtime containers into its value, which is a structure JSON of RuntimeContainerMetaSet type. RuntimeContainerMetaKey = "apps.kruise.io/runtime-containers-meta" + + // InPlaceUpdatePodRestartKey records the count of pod restarts + InPlaceUpdatePodRestartKey = "apps.kruise.io/inplace-update-pod-restart-count" + + // InPlaceUpdateContainersRestartKey records the count of containers restarts + InPlaceUpdateContainersRestartKey = "apps.kruise.io/inplace-update-containers-restart-count" ) // InPlaceUpdateState records latest inplace-update state, including old statuses of containers. @@ -101,6 +107,16 @@ type InPlaceUpdateStrategy struct { GracePeriodSeconds int32 `json:"gracePeriodSeconds,omitempty"` } +// InPlaceUpdateContainerRestartCount record the restart count of containers +type InPlaceUpdateContainerRestartCount struct { + // Revision is the updated revision hash. + Revision string `json:"revision,omitempty"` + // RestartCount is the container restart count + RestartCount int `json:"restartCount,omitempty"` + // Timestamp is the time for this update + Timestamp metav1.Time `json:"timestamp"` +} + func GetInPlaceUpdateState(obj metav1.Object) (string, bool) { if v, ok := obj.GetAnnotations()[InPlaceUpdateStateKey]; ok { return v, ok diff --git a/pkg/controller/cloneset/sync/cloneset_update.go b/pkg/controller/cloneset/sync/cloneset_update.go index 40089214ec..517aa6c27f 100644 --- a/pkg/controller/cloneset/sync/cloneset_update.go +++ b/pkg/controller/cloneset/sync/cloneset_update.go @@ -50,7 +50,6 @@ func (c *realControl) Update(cs *appsv1alpha1.CloneSet, key := clonesetutils.GetControllerKey(cs) coreControl := clonesetcore.New(cs) - // 1. refresh states for all pods var modified bool for _, pod := range pods { @@ -249,7 +248,6 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc break } } - if c.inplaceControl.CanUpdateInPlace(oldRevision, updateRevision, coreControl.GetUpdateOptions()) { switch state := lifecycle.GetPodLifecycleState(pod); state { case "", appspub.LifecycleStatePreparingNormal, appspub.LifecycleStateNormal: diff --git a/pkg/controller/cloneset/sync/cloneset_update_test.go b/pkg/controller/cloneset/sync/cloneset_update_test.go index 17add8342a..7ed983d31d 100644 --- a/pkg/controller/cloneset/sync/cloneset_update_test.go +++ b/pkg/controller/cloneset/sync/cloneset_update_test.go @@ -307,7 +307,12 @@ func TestUpdate(t *testing.T) { UpdateTimestamp: now, LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}}, ContainerBatchesRecord: []appspub.InPlaceUpdateContainerBatch{{Timestamp: now, Containers: []string{"c1"}}}, - })}, + }), + appspub.InPlaceUpdatePodRestartKey: "1", + appspub.InPlaceUpdateContainersRestartKey: util.DumpJSON(map[string]appspub.InPlaceUpdateContainerRestartCount{ + "c1": {RestartCount: 1, Revision: "rev_new", Timestamp: now}, + }), + }, ResourceVersion: "2", }, Spec: v1.PodSpec{ @@ -555,6 +560,10 @@ func TestUpdate(t *testing.T) { LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}}, ContainerBatchesRecord: []appspub.InPlaceUpdateContainerBatch{{Timestamp: now, Containers: []string{"c1"}}}, }), + appspub.InPlaceUpdatePodRestartKey: "1", + appspub.InPlaceUpdateContainersRestartKey: util.DumpJSON(map[string]appspub.InPlaceUpdateContainerRestartCount{ + "c1": {RestartCount: 1, Revision: "rev_new", Timestamp: now}, + }), }, ResourceVersion: "1", }, diff --git a/pkg/util/inplaceupdate/inplace_update.go b/pkg/util/inplaceupdate/inplace_update.go index 81b8966ef9..a8da972418 100644 --- a/pkg/util/inplaceupdate/inplace_update.go +++ b/pkg/util/inplaceupdate/inplace_update.go @@ -20,13 +20,10 @@ import ( "encoding/json" "fmt" "regexp" + "strconv" "strings" "time" - appspub "github.com/openkruise/kruise/apis/apps/pub" - "github.com/openkruise/kruise/pkg/util" - "github.com/openkruise/kruise/pkg/util/podadapter" - "github.com/openkruise/kruise/pkg/util/revisionadapter" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,6 +33,11 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + + appspub "github.com/openkruise/kruise/apis/apps/pub" + "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/kruise/pkg/util/podadapter" + "github.com/openkruise/kruise/pkg/util/revisionadapter" ) var ( @@ -138,7 +140,6 @@ func (c *realControl) Refresh(pod *v1.Pod, opts *UpdateOptions) RefreshResult { klog.V(5).Infof("Pod %s/%s in-place update pre-check not passed: %v", pod.Namespace, pod.Name, checkErr) return RefreshResult{} } - // do update the next containers if updated, err := c.updateNextBatch(pod, opts); err != nil { return RefreshResult{RefreshErr: err} @@ -216,13 +217,15 @@ func (c *realControl) finishGracePeriod(pod *v1.Pod, opts *UpdateOptions) (time. delayDuration = roundupSeconds(graceDuration - span) return nil } - if clone, err = opts.PatchSpecToPod(clone, &spec, &updateState); err != nil { return err } + // record restart count of the pod(containers) + if spec.ContainerImages != nil || spec.UpdateEnvFromMetadata { + recordPodAndContainersRestartCount(clone, &spec, updateState.Revision) + } appspub.RemoveInPlaceUpdateGrace(clone) } - _, err = c.podAdapter.UpdatePod(clone) return err }) @@ -258,7 +261,10 @@ func (c *realControl) updateNextBatch(pod *v1.Pod, opts *UpdateOptions) (bool, e if clone, err = opts.PatchSpecToPod(clone, &spec, &state); err != nil { return err } - + // record restart count of the pod(containers) + if spec.ContainerImages != nil || spec.UpdateEnvFromMetadata { + recordPodAndContainersRestartCount(clone, &spec, state.Revision) + } updated = true _, err = c.podAdapter.UpdatePod(clone) return err @@ -273,7 +279,6 @@ func (c *realControl) CanUpdateInPlace(oldRevision, newRevision *apps.Controller func (c *realControl) Update(pod *v1.Pod, oldRevision, newRevision *apps.ControllerRevision, opts *UpdateOptions) UpdateResult { opts = SetOptionsDefaults(opts) - // 1. calculate inplace update spec spec := opts.CalculateSpec(oldRevision, newRevision, opts) if spec == nil { @@ -338,12 +343,15 @@ func (c *realControl) updatePodInPlace(pod *v1.Pod, spec *UpdateSpec, opts *Upda if clone, err = opts.PatchSpecToPod(clone, spec, &inPlaceUpdateState); err != nil { return err } + // record restart count of the pod(containers) + if spec.ContainerImages != nil || spec.UpdateEnvFromMetadata { + recordPodAndContainersRestartCount(clone, spec, inPlaceUpdateState.Revision) + } appspub.RemoveInPlaceUpdateGrace(clone) } else { inPlaceUpdateSpecJSON, _ := json.Marshal(spec) clone.Annotations[appspub.InPlaceUpdateGraceKey] = string(inPlaceUpdateSpecJSON) } - newPod, updateErr := c.podAdapter.UpdatePod(clone) if updateErr == nil { newResourceVersion = newPod.ResourceVersion @@ -419,3 +427,55 @@ func hasEqualCondition(pod *v1.Pod, newCondition *v1.PodCondition) bool { oldCondition.Reason == newCondition.Reason && oldCondition.Message == newCondition.Message return isEqual } + +// recordPodAndContainersRestartCount record the count of pod(containers) restarts +func recordPodAndContainersRestartCount(pod *v1.Pod, spec *UpdateSpec, revision string) { + var currentPodRestartCount int64 + containersRestartCount := make(map[string]appspub.InPlaceUpdateContainerRestartCount) + //revision := pod.GetLabels()["controller-revision-hash"] + + if v, ok := pod.Annotations[appspub.InPlaceUpdatePodRestartKey]; ok { + currentPodRestartCount, _ = strconv.ParseInt(v, 10, 64) + } + + if v, ok := pod.Annotations[appspub.InPlaceUpdateContainersRestartKey]; ok { + _ = json.Unmarshal([]byte(v), &containersRestartCount) + } + calculateRestartCountFunc := func(cname string) { + containerRestartCount, ok := containersRestartCount[cname] + if ok && revision != containerRestartCount.Revision { + containerRestartCount.RestartCount += 1 + containerRestartCount.Revision = revision + containerRestartCount.Timestamp = metav1.NewTime(Clock.Now()) + currentPodRestartCount += 1 // pod restart + } else if !ok { + containerRestartCount = appspub.InPlaceUpdateContainerRestartCount{ + Revision: revision, + RestartCount: 1, + Timestamp: metav1.NewTime(Clock.Now()), + } + currentPodRestartCount += 1 // pod restart + } + containersRestartCount[cname] = containerRestartCount // containers restart + } + + containers := make(map[string]bool) + if spec.ContainerImages != nil { + for cname := range spec.ContainerImages { + containers[cname] = true + calculateRestartCountFunc(cname) + } + } + + if spec.UpdateEnvFromMetadata { + for cname := range spec.ContainerRefMetadata { + if !containers[cname] { + calculateRestartCountFunc(cname) + } + } + } + + containersRestartCountJson, _ := json.Marshal(containersRestartCount) + pod.Annotations[appspub.InPlaceUpdateContainersRestartKey] = string(containersRestartCountJson) + pod.Annotations[appspub.InPlaceUpdatePodRestartKey] = strconv.FormatInt(currentPodRestartCount, 10) +} diff --git a/pkg/util/inplaceupdate/inplace_update_test.go b/pkg/util/inplaceupdate/inplace_update_test.go index 75353a8ae9..224887b47e 100644 --- a/pkg/util/inplaceupdate/inplace_update_test.go +++ b/pkg/util/inplaceupdate/inplace_update_test.go @@ -23,11 +23,6 @@ import ( "testing" "time" - appspub "github.com/openkruise/kruise/apis/apps/pub" - "github.com/openkruise/kruise/pkg/features" - "github.com/openkruise/kruise/pkg/util" - utilfeature "github.com/openkruise/kruise/pkg/util/feature" - "github.com/openkruise/kruise/pkg/util/revisionadapter" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,6 +30,12 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + appspub "github.com/openkruise/kruise/apis/apps/pub" + "github.com/openkruise/kruise/pkg/features" + "github.com/openkruise/kruise/pkg/util" + utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "github.com/openkruise/kruise/pkg/util/revisionadapter" ) func TestCalculateInPlaceUpdateSpec(t *testing.T) { @@ -451,6 +452,10 @@ func TestRefresh(t *testing.T) { LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "img01"}}, ContainerBatchesRecord: []appspub.InPlaceUpdateContainerBatch{{Timestamp: aHourAgo, Containers: []string{"main"}}}, }), + appspub.InPlaceUpdatePodRestartKey: "1", + appspub.InPlaceUpdateContainersRestartKey: util.DumpJSON(map[string]appspub.InPlaceUpdateContainerRestartCount{ + "main": {RestartCount: 1, Revision: "new-revision", Timestamp: aHourAgo}, + }), }, ResourceVersion: "1", }, @@ -588,6 +593,10 @@ func TestRefresh(t *testing.T) { LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "c1-img1-ID"}, "c2": {ImageID: "c2-img1-ID"}}, ContainerBatchesRecord: []appspub.InPlaceUpdateContainerBatch{{Timestamp: aHourAgo, Containers: []string{"c1"}}, {Timestamp: aHourAgo, Containers: []string{"c2"}}}, }), + appspub.InPlaceUpdatePodRestartKey: "1", + appspub.InPlaceUpdateContainersRestartKey: util.DumpJSON(map[string]appspub.InPlaceUpdateContainerRestartCount{ + "c2": {RestartCount: 1, Revision: "new-revision", Timestamp: aHourAgo}, + }), }, }, Spec: v1.PodSpec{ From 121eb2860fd9362ab636cae28e3faa82407bebad Mon Sep 17 00:00:00 2001 From: Liu Zhenwei Date: Thu, 24 Mar 2022 15:42:12 +0800 Subject: [PATCH 2/4] move if statement into function Signed-off-by: Liu Zhenwei Signed-off-by: liuzhenwei --- pkg/util/inplaceupdate/inplace_update.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pkg/util/inplaceupdate/inplace_update.go b/pkg/util/inplaceupdate/inplace_update.go index a8da972418..12c43a57d5 100644 --- a/pkg/util/inplaceupdate/inplace_update.go +++ b/pkg/util/inplaceupdate/inplace_update.go @@ -221,9 +221,7 @@ func (c *realControl) finishGracePeriod(pod *v1.Pod, opts *UpdateOptions) (time. return err } // record restart count of the pod(containers) - if spec.ContainerImages != nil || spec.UpdateEnvFromMetadata { - recordPodAndContainersRestartCount(clone, &spec, updateState.Revision) - } + recordPodAndContainersRestartCount(clone, &spec, updateState.Revision) appspub.RemoveInPlaceUpdateGrace(clone) } _, err = c.podAdapter.UpdatePod(clone) @@ -262,9 +260,7 @@ func (c *realControl) updateNextBatch(pod *v1.Pod, opts *UpdateOptions) (bool, e return err } // record restart count of the pod(containers) - if spec.ContainerImages != nil || spec.UpdateEnvFromMetadata { - recordPodAndContainersRestartCount(clone, &spec, state.Revision) - } + recordPodAndContainersRestartCount(clone, &spec, state.Revision) updated = true _, err = c.podAdapter.UpdatePod(clone) return err @@ -344,9 +340,7 @@ func (c *realControl) updatePodInPlace(pod *v1.Pod, spec *UpdateSpec, opts *Upda return err } // record restart count of the pod(containers) - if spec.ContainerImages != nil || spec.UpdateEnvFromMetadata { - recordPodAndContainersRestartCount(clone, spec, inPlaceUpdateState.Revision) - } + recordPodAndContainersRestartCount(clone, spec, inPlaceUpdateState.Revision) appspub.RemoveInPlaceUpdateGrace(clone) } else { inPlaceUpdateSpecJSON, _ := json.Marshal(spec) @@ -430,6 +424,9 @@ func hasEqualCondition(pod *v1.Pod, newCondition *v1.PodCondition) bool { // recordPodAndContainersRestartCount record the count of pod(containers) restarts func recordPodAndContainersRestartCount(pod *v1.Pod, spec *UpdateSpec, revision string) { + if spec.ContainerImages == nil && !spec.UpdateEnvFromMetadata { + return + } var currentPodRestartCount int64 containersRestartCount := make(map[string]appspub.InPlaceUpdateContainerRestartCount) //revision := pod.GetLabels()["controller-revision-hash"] From b4bd74c7dac95102e85be99c5e3c532892eb6a2f Mon Sep 17 00:00:00 2001 From: Liu Zhenwei Date: Fri, 25 Mar 2022 19:54:04 +0800 Subject: [PATCH 3/4] remove comment and handle the error of Unmarshal Signed-off-by: Liu Zhenwei Signed-off-by: liuzhenwei --- apis/apps/pub/inplace_update.go | 2 +- pkg/util/inplaceupdate/inplace_update.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apis/apps/pub/inplace_update.go b/apis/apps/pub/inplace_update.go index 768ae4bcf5..e08efd4675 100644 --- a/apis/apps/pub/inplace_update.go +++ b/apis/apps/pub/inplace_update.go @@ -114,7 +114,7 @@ type InPlaceUpdateContainerRestartCount struct { // RestartCount is the container restart count RestartCount int `json:"restartCount,omitempty"` // Timestamp is the time for this update - Timestamp metav1.Time `json:"timestamp"` + Timestamp metav1.Time `json:"timestamp,omitempty"` } func GetInPlaceUpdateState(obj metav1.Object) (string, bool) { diff --git a/pkg/util/inplaceupdate/inplace_update.go b/pkg/util/inplaceupdate/inplace_update.go index 12c43a57d5..f3e7fd0c84 100644 --- a/pkg/util/inplaceupdate/inplace_update.go +++ b/pkg/util/inplaceupdate/inplace_update.go @@ -90,7 +90,6 @@ type UpdateSpec struct { OldTemplate *v1.PodTemplateSpec `json:"oldTemplate,omitempty"` NewTemplate *v1.PodTemplateSpec `json:"newTemplate,omitempty"` } - type realControl struct { podAdapter podadapter.Adapter revisionAdapter revisionadapter.Interface @@ -429,14 +428,15 @@ func recordPodAndContainersRestartCount(pod *v1.Pod, spec *UpdateSpec, revision } var currentPodRestartCount int64 containersRestartCount := make(map[string]appspub.InPlaceUpdateContainerRestartCount) - //revision := pod.GetLabels()["controller-revision-hash"] if v, ok := pod.Annotations[appspub.InPlaceUpdatePodRestartKey]; ok { currentPodRestartCount, _ = strconv.ParseInt(v, 10, 64) } if v, ok := pod.Annotations[appspub.InPlaceUpdateContainersRestartKey]; ok { - _ = json.Unmarshal([]byte(v), &containersRestartCount) + if err := json.Unmarshal([]byte(v), &containersRestartCount); err != nil { + return + } } calculateRestartCountFunc := func(cname string) { containerRestartCount, ok := containersRestartCount[cname] From bc7082315bfb4a8c6a811bc5d44265eb91c0a40b Mon Sep 17 00:00:00 2001 From: Liu Zhenwei Date: Fri, 7 Oct 2022 22:16:39 +0800 Subject: [PATCH 4/4] make generate manifests Signed-off-by: liuzhenwei --- apis/apps/pub/zz_generated.deepcopy.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apis/apps/pub/zz_generated.deepcopy.go b/apis/apps/pub/zz_generated.deepcopy.go index cda5d19b79..aa8131a746 100644 --- a/apis/apps/pub/zz_generated.deepcopy.go +++ b/apis/apps/pub/zz_generated.deepcopy.go @@ -46,6 +46,22 @@ func (in *InPlaceUpdateContainerBatch) DeepCopy() *InPlaceUpdateContainerBatch { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InPlaceUpdateContainerRestartCount) DeepCopyInto(out *InPlaceUpdateContainerRestartCount) { + *out = *in + in.Timestamp.DeepCopyInto(&out.Timestamp) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InPlaceUpdateContainerRestartCount. +func (in *InPlaceUpdateContainerRestartCount) DeepCopy() *InPlaceUpdateContainerRestartCount { + if in == nil { + return nil + } + out := new(InPlaceUpdateContainerRestartCount) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InPlaceUpdateContainerStatus) DeepCopyInto(out *InPlaceUpdateContainerStatus) { *out = *in