Skip to content

Commit b41a89a

Browse files
chrisliu1995chrisliu1995
committed
sync podCondition when probe message of NodePodProbe changed
Signed-off-by: ChrisLiu <[email protected]>
1 parent fa9a9a0 commit b41a89a

File tree

5 files changed

+322
-2
lines changed

5 files changed

+322
-2
lines changed

Diff for: ‎pkg/controller/nodepodprobe/node_pod_probe_controller.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,7 @@ func (r *ReconcileNodePodProbe) updatePodProbeStatus(pod *corev1.Pod, status app
333333
oldStatus := podClone.Status.DeepCopy()
334334
for i := range probeConditions {
335335
condition := probeConditions[i]
336-
util.SetPodCondition(podClone, condition)
336+
util.SetPodConditionIfMsgChanged(podClone, condition)
337337
}
338338
oldMetadata := podClone.ObjectMeta.DeepCopy()
339339
if podClone.Annotations == nil {

Diff for: ‎pkg/util/pods.go

+12
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,18 @@ func SetPodCondition(pod *v1.Pod, condition v1.PodCondition) {
304304
pod.Status.Conditions = append(pod.Status.Conditions, condition)
305305
}
306306

307+
func SetPodConditionIfMsgChanged(pod *v1.Pod, condition v1.PodCondition) {
308+
for i, c := range pod.Status.Conditions {
309+
if c.Type == condition.Type {
310+
if c.Status != condition.Status || c.Message != condition.Message {
311+
pod.Status.Conditions[i] = condition
312+
}
313+
return
314+
}
315+
}
316+
pod.Status.Conditions = append(pod.Status.Conditions, condition)
317+
}
318+
307319
func SetPodReadyCondition(pod *v1.Pod) {
308320
podReady := GetCondition(pod, v1.PodReady)
309321
if podReady == nil {

Diff for: ‎pkg/util/pods_test.go

+124
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package util
1818

1919
import (
20+
"reflect"
2021
"testing"
2122

2223
v1 "k8s.io/api/core/v1"
@@ -102,3 +103,126 @@ func TestMergeVolumes(t *testing.T) {
102103
}
103104
}
104105
}
106+
107+
func TestSetPodConditionIfMsgChanged(t *testing.T) {
108+
tests := []struct {
109+
pod *v1.Pod
110+
condition v1.PodCondition
111+
conditions []v1.PodCondition
112+
}{
113+
// case 0: existed condition status changed
114+
{
115+
pod: &v1.Pod{
116+
Status: v1.PodStatus{
117+
Conditions: []v1.PodCondition{
118+
{
119+
Type: "type-0",
120+
Status: v1.ConditionTrue,
121+
Message: "Msg-0",
122+
},
123+
{
124+
Type: "type-1",
125+
Status: v1.ConditionFalse,
126+
},
127+
},
128+
},
129+
},
130+
condition: v1.PodCondition{
131+
Type: "type-0",
132+
Status: v1.ConditionFalse,
133+
},
134+
conditions: []v1.PodCondition{
135+
{
136+
Type: "type-0",
137+
Status: v1.ConditionFalse,
138+
},
139+
{
140+
Type: "type-1",
141+
Status: v1.ConditionFalse,
142+
},
143+
},
144+
},
145+
146+
// case 1: add a new condition
147+
{
148+
pod: &v1.Pod{
149+
Status: v1.PodStatus{
150+
Conditions: []v1.PodCondition{
151+
{
152+
Type: "type-0",
153+
Status: v1.ConditionTrue,
154+
Message: "Msg-0",
155+
},
156+
{
157+
Type: "type-1",
158+
Status: v1.ConditionFalse,
159+
},
160+
},
161+
},
162+
},
163+
condition: v1.PodCondition{
164+
Type: "type-2",
165+
Status: v1.ConditionFalse,
166+
},
167+
conditions: []v1.PodCondition{
168+
{
169+
Type: "type-0",
170+
Status: v1.ConditionTrue,
171+
Message: "Msg-0",
172+
},
173+
{
174+
Type: "type-1",
175+
Status: v1.ConditionFalse,
176+
},
177+
{
178+
Type: "type-2",
179+
Status: v1.ConditionFalse,
180+
},
181+
},
182+
},
183+
184+
// case 2: existed condition status not changed, but message changed
185+
{
186+
pod: &v1.Pod{
187+
Status: v1.PodStatus{
188+
Conditions: []v1.PodCondition{
189+
{
190+
Type: "type-0",
191+
Status: v1.ConditionTrue,
192+
Message: "Msg-0",
193+
},
194+
{
195+
Type: "type-1",
196+
Status: v1.ConditionFalse,
197+
},
198+
},
199+
},
200+
},
201+
condition: v1.PodCondition{
202+
Type: "type-0",
203+
Status: v1.ConditionTrue,
204+
Message: "Msg-Changed",
205+
},
206+
conditions: []v1.PodCondition{
207+
{
208+
Type: "type-0",
209+
Status: v1.ConditionTrue,
210+
Message: "Msg-Changed",
211+
},
212+
{
213+
Type: "type-1",
214+
Status: v1.ConditionFalse,
215+
},
216+
},
217+
},
218+
}
219+
220+
for i, test := range tests {
221+
expect := test.conditions
222+
SetPodConditionIfMsgChanged(test.pod, test.condition)
223+
actual := test.pod.Status.Conditions
224+
if !reflect.DeepEqual(expect, actual) {
225+
t.Fatalf("case %d: expect Conditions(%s), but get %s", i, expect, actual)
226+
}
227+
}
228+
}

Diff for: ‎test/e2e/apps/podprobemarker.go

+113
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ package apps
1919
import (
2020
"context"
2121
"fmt"
22+
"k8s.io/apimachinery/pkg/util/wait"
23+
"strings"
2224
"time"
2325

2426
"github.com/onsi/ginkgo"
@@ -273,5 +275,116 @@ var _ = SIGDescribe("PodProbeMarker", func() {
273275
gomega.Expect(npp.Spec.PodProbes).To(gomega.HaveLen(0))
274276
}
275277
})
278+
279+
ginkgo.It("pod probe marker test2", func() {
280+
nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
281+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
282+
nodeLen := len(nodeList.Items)
283+
if nodeLen == 0 {
284+
ginkgo.By("pod probe markers list nodeList is zero")
285+
return
286+
}
287+
nppList, err := kc.AppsV1alpha1().NodePodProbes().List(context.TODO(), metav1.ListOptions{})
288+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
289+
gomega.Expect(nppList.Items).To(gomega.HaveLen(nodeLen))
290+
291+
// create statefulset
292+
sts := tester.NewStatefulSetWithProbeImg(ns, randStr)
293+
// For heterogeneous scenario like edge cluster, I want to deploy a Pod for each Node to verify that the functionality works
294+
sts.Spec.Template.Spec.TopologySpreadConstraints = []v1.TopologySpreadConstraint{
295+
{
296+
LabelSelector: sts.Spec.Selector,
297+
MaxSkew: 1,
298+
TopologyKey: "kubernetes.io/hostname",
299+
WhenUnsatisfiable: v1.ScheduleAnyway,
300+
},
301+
}
302+
sts.Spec.Replicas = utilpointer.Int32Ptr(int32(nodeLen))
303+
ginkgo.By(fmt.Sprintf("Create statefulset(%s/%s)", sts.Namespace, sts.Name))
304+
tester.CreateStatefulSet(sts)
305+
306+
// create pod probe marker
307+
ppm := tester.NewPodProbeMarkerWithProbeImg(ns, randStr)
308+
_, err = kc.AppsV1alpha1().PodProbeMarkers(ns).Create(context.TODO(), &ppm, metav1.CreateOptions{})
309+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
310+
311+
// Firstly, probe.sh return false
312+
err = wait.PollImmediate(1*time.Second, 10*time.Second,
313+
func() (done bool, err error) {
314+
pods, err := tester.ListActivePods(ns)
315+
if err != nil {
316+
return false, err
317+
}
318+
if len(pods) != nodeLen {
319+
return false, nil
320+
}
321+
322+
for _, pod := range pods {
323+
condition := util.GetCondition(pod, "game.kruise.io/healthy")
324+
if condition == nil {
325+
return false, nil
326+
}
327+
if condition.Status != v1.ConditionFalse {
328+
return false, nil
329+
}
330+
}
331+
return true, nil
332+
})
333+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
334+
335+
// Secondly, probe.sh return true & msg is 'data'
336+
err = wait.PollImmediate(1*time.Second, 10*time.Second,
337+
func() (done bool, err error) {
338+
pods, err := tester.ListActivePods(ns)
339+
if err != nil {
340+
return false, err
341+
}
342+
if len(pods) != nodeLen {
343+
return false, fmt.Errorf("the num of pods(%d) is not same as num of nodes(%d)", len(pods), nodeLen)
344+
}
345+
346+
for _, pod := range pods {
347+
condition := util.GetCondition(pod, "game.kruise.io/healthy")
348+
if condition == nil {
349+
return false, nil
350+
}
351+
if condition.Status != v1.ConditionTrue {
352+
return false, nil
353+
}
354+
if !strings.Contains(condition.Message, "data") {
355+
return false, nil
356+
}
357+
}
358+
return true, nil
359+
})
360+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
361+
362+
// Thirdly, probe.sh return true & msg is 'gate'
363+
err = wait.PollImmediate(1*time.Second, 10*time.Second,
364+
func() (done bool, err error) {
365+
pods, err := tester.ListActivePods(ns)
366+
if err != nil {
367+
return false, err
368+
}
369+
if len(pods) != nodeLen {
370+
return false, fmt.Errorf("the num of pods(%d) is not same as num of nodes(%d)", len(pods), nodeLen)
371+
}
372+
373+
for _, pod := range pods {
374+
condition := util.GetCondition(pod, "game.kruise.io/healthy")
375+
if condition == nil {
376+
return false, nil
377+
}
378+
if condition.Status != v1.ConditionTrue {
379+
return false, nil
380+
}
381+
if !strings.Contains(condition.Message, "gate") {
382+
return false, nil
383+
}
384+
}
385+
return true, nil
386+
})
387+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
388+
})
276389
})
277390
})

Diff for: ‎test/e2e/framework/pod_probe_marker_util.go

+72-1
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,77 @@ func (s *PodProbeMarkerTester) NewBaseStatefulSet(namespace, randStr string) *ap
177177
}
178178
}
179179

180+
func (s *PodProbeMarkerTester) NewPodProbeMarkerWithProbeImg(ns, randStr string) appsv1alpha1.PodProbeMarker {
181+
return appsv1alpha1.PodProbeMarker{
182+
ObjectMeta: metav1.ObjectMeta{
183+
Name: "ppm-minecraft",
184+
Namespace: ns,
185+
},
186+
Spec: appsv1alpha1.PodProbeMarkerSpec{
187+
Selector: &metav1.LabelSelector{
188+
MatchLabels: map[string]string{
189+
"app": fmt.Sprintf("probe-%s", randStr),
190+
},
191+
},
192+
Probes: []appsv1alpha1.PodContainerProbe{
193+
{
194+
Name: "healthy",
195+
ContainerName: "minecraft",
196+
Probe: appsv1alpha1.ContainerProbeSpec{
197+
Probe: corev1.Probe{
198+
ProbeHandler: corev1.ProbeHandler{
199+
Exec: &corev1.ExecAction{
200+
Command: []string{"bash", "./probe.sh"},
201+
},
202+
},
203+
},
204+
},
205+
PodConditionType: "game.kruise.io/healthy",
206+
},
207+
},
208+
},
209+
}
210+
}
211+
212+
func (s *PodProbeMarkerTester) NewStatefulSetWithProbeImg(namespace, randStr string) *appsv1beta1.StatefulSet {
213+
return &appsv1beta1.StatefulSet{
214+
TypeMeta: metav1.TypeMeta{
215+
Kind: "StatefulSet",
216+
APIVersion: "apps.kruise.io/v1beta1",
217+
},
218+
ObjectMeta: metav1.ObjectMeta{
219+
Name: "stateful-test",
220+
Namespace: namespace,
221+
},
222+
Spec: appsv1beta1.StatefulSetSpec{
223+
PodManagementPolicy: apps.ParallelPodManagement,
224+
ServiceName: "fake-service",
225+
Replicas: utilpointer.Int32Ptr(2),
226+
Selector: &metav1.LabelSelector{
227+
MatchLabels: map[string]string{
228+
"app": fmt.Sprintf("probe-%s", randStr),
229+
},
230+
},
231+
Template: corev1.PodTemplateSpec{
232+
ObjectMeta: metav1.ObjectMeta{
233+
Labels: map[string]string{
234+
"app": fmt.Sprintf("probe-%s", randStr),
235+
},
236+
},
237+
Spec: corev1.PodSpec{
238+
Containers: []corev1.Container{
239+
{
240+
Name: "minecraft",
241+
Image: "openkruise/minecraft-demo:probe-v0",
242+
ImagePullPolicy: corev1.PullIfNotPresent,
243+
},
244+
},
245+
},
246+
},
247+
},
248+
}
249+
}
250+
180251
func (s *PodProbeMarkerTester) CreateStatefulSet(sts *appsv1beta1.StatefulSet) {
181252
Logf("create sts(%s/%s)", sts.Namespace, sts.Name)
182253
_, err := s.kc.AppsV1beta1().StatefulSets(sts.Namespace).Create(context.TODO(), sts, metav1.CreateOptions{})
@@ -185,7 +256,7 @@ func (s *PodProbeMarkerTester) CreateStatefulSet(sts *appsv1beta1.StatefulSet) {
185256
}
186257

187258
func (s *PodProbeMarkerTester) WaitForStatefulSetRunning(sts *appsv1beta1.StatefulSet) {
188-
pollErr := wait.PollImmediate(time.Second, time.Minute,
259+
pollErr := wait.PollImmediate(time.Second, 2*time.Minute,
189260
func() (bool, error) {
190261
inner, err := s.kc.AppsV1beta1().StatefulSets(sts.Namespace).Get(context.TODO(), sts.Name, metav1.GetOptions{})
191262
if err != nil {

0 commit comments

Comments
 (0)