vm pod and cdi pod frequencily recreate.maybe apiserver is on pressure.the capi-system pod is flushing #12911
Description
What steps did you take and what happened?
^C[root@kamaji-manager-01-a ~]# kubectl get pod -n sveltos-member-02 -w
NAME READY STATUS RESTARTS AGE
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-zt89m-root-volume 1/1 Running 0 3m9s
sveltos-member-02-dcbv7-bff898f99-lg8qg 4/4 Running 28 (44s ago) 80m
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-l8rk4-89wtc 2/2 Running 0 3m32s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-pjqbv-cqzbz 2/2 Running 0 3m32s
sveltos-member-02-dcbv7-bff898f99-lg8qg 3/4 Error 28 (56s ago) 81m
sveltos-member-02-dcbv7-bff898f99-lg8qg 3/4 CrashLoopBackOff 28 (4s ago) 81m
sveltos-member-02-dcbv7-bff898f99-lg8qg 3/4 Running 29 (48s ago) 81m
sveltos-member-02-dcbv7-bff898f99-lg8qg 3/4 Running 29 (54s ago) 82m
sveltos-member-02-dcbv7-bff898f99-lg8qg 4/4 Running 29 (55s ago) 82m
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-zt89m-root-volume 0/1 Completed 0 4m48s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-zt89m-root-volume 0/1 Completed 0 4m50s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-zt89m-root-volume 0/1 Terminating 0 4m50s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-zt89m-root-volume 0/1 Terminating 0 4m50s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 0/2 Pending 0 0s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 0/2 Pending 0 0s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 0/2 Pending 0 0s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 0/2 ContainerCreating 0 0s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 0/2 ContainerCreating 0 3s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 0/2 ContainerCreating 0 4s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 2/2 Running 0 5s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 2/2 Running 0 5s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-pjqbv-cqzbz 2/2 Terminating 0 5m50s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-l8rk4-89wtc 2/2 Terminating 0 5m51s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 Pending 0 1s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 Pending 0 1s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 ContainerCreating 0 1s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 ContainerCreating 0 11s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 ContainerCreating 0 11s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 Running 0 12s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 1/1 Running 0 13s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 Pending 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 Pending 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 ContainerCreating 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 ContainerCreating 0 12s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 ContainerCreating 0 12s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 Running 0 13s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 1/1 Running 0 14s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-pjqbv-cqzbz 0/2 Terminating 0 6m30s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-pjqbv-cqzbz 0/2 Terminating 0 6m31s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-pjqbv-cqzbz 0/2 Terminating 0 6m31s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-pjqbv-cqzbz 0/2 Terminating 0 6m31s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-l8rk4-89wtc 0/2 Terminating 0 6m32s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-l8rk4-89wtc 0/2 Terminating 0 6m33s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-l8rk4-89wtc 0/2 Terminating 0 6m33s
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-l8rk4-89wtc 0/2 Terminating 0 6m33s
sveltos-member-02-dcbv7-bff898f99-lg8qg 3/4 Running 29 (5m35s ago) 86m
sveltos-member-02-dcbv7-bff898f99-lg8qg 4/4 Running 29 (5m44s ago) 86m
sveltos-member-02-dcbv7-bff898f99-lg8qg 3/4 Running 29 (8m25s ago) 89m
sveltos-member-02-dcbv7-bff898f99-lg8qg 4/4 Running 29 (8m25s ago) 89m
^C[root@kamaji-manager-01-a ~]# kubectl get pod -n sveltos-member-02 -w
NAME READY STATUS RESTARTS AGE
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 1/1 Running 0 8m42s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 1/1 Running 0 8m57s
sveltos-member-02-dcbv7-bff898f99-lg8qg 4/4 Running 29 (11m ago) 92m
virt-launcher-sveltos-member-02-md-0-gg52k-qplpm-zt89m-tg8vm 2/2 Running 0 9m37s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 1/1 Terminating 0 10m
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 1/1 Terminating 0 9m45s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-64kw9-root-volume 0/1 Pending 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-64kw9-root-volume 0/1 Pending 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-64kw9-root-volume 0/1 ContainerCreating 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 Terminating 0 10m
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 Terminating 0 9m49s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 Terminating 0 10m
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 Terminating 0 10m
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-p9cjr-root-volume 0/1 Terminating 0 10m
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 Terminating 0 9m49s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 Terminating 0 9m49s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-mq9jl-root-volume 0/1 Terminating 0 9m49s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-k9gxt-root-volume 0/1 Pending 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-k9gxt-root-volume 0/1 Pending 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-k9gxt-root-volume 0/1 ContainerCreating 0 0s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-64kw9-root-volume 0/1 ContainerCreating 0 40s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-64kw9-root-volume 0/1 ContainerCreating 0 40s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-64kw9-root-volume 0/1 Running 0 40s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-64kw9-root-volume 1/1 Running 0 42s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-k9gxt-root-volume 0/1 ContainerCreating 0 61s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-k9gxt-root-volume 0/1 ContainerCreating 0 61s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-k9gxt-root-volume 0/1 Running 0 62s
cdi-upload-sveltos-member-02-md-0-gg52k-qplpm-k9gxt-root-volume 1/1 Running 0 63s
cat << EOF|kubectl apply -f -
apiVersion: flowcontrol.apiserver.k8s.io/v1
kind: PriorityLevelConfiguration
metadata:
name: topology-update-low-priority
spec:
type: Limited
limited:
nominalConcurrencyShares: 4
lendablePercent: 0
borrowingLimitPercent: 0
limitResponse:
type: Queue
queuing:
queues: 2
queueLengthLimit: 5
handSize: 1
---
apiVersion: flowcontrol.apiserver.k8s.io/v1
kind: FlowSchema
metadata:
name: topology-update-requests
spec:
matchingPrecedence: 9000
priorityLevelConfiguration:
name: topology-update-low-priority
rules:
- subjects:
- kind: Group
group:
name: system:authenticated
resourceRules:
- apiGroups: ['cluster.x-k8s.io']
namespaces: ['*']
resources: ['clusters', 'kubevirtclusters', 'kamajicontrolplanes']
verbs: ['update', 'patch', 'list', 'watch']
EOF
I1030 02:07:28.317238 1 recorder.go:104] "Updated KamajiControlPlane \"sveltos-member-02/sveltos-member-02-dcbv7\"" logger="events" type="Normal" object={"kind":"Cluster","namespace":"sveltos-member-02","name":"sveltos-member-02","uid":"641a4ac5-77b3-4e6d-9167-414fef24c1cd","apiVersion":"cluster.x-k8s.io/v1beta2","resourceVersion":"2529272"} reason="TopologyUpdate"
I1030 02:07:28.381551 1 reconcile_state.go:1238] "Patching KubevirtCluster" controller="topology/cluster" controllerGroup="cluster.x-k8s.io" controllerKind="Cluster" Cluster="sveltos-member-02/sveltos-member-02" namespace="sveltos-member-02" name="sveltos-member-02" reconcileID="95b4a1c6-12ad-4bd2-81a6-bbbbbc207003" KubevirtCluster="sveltos-member-02/sveltos-member-02-kwwcd" KubevirtCluster="sveltos-member-02/sveltos-member-02-kwwcd" diff="{\"spec\":{\"sshKeys\":{\"dataSecretName\":\"sveltos-member-02-ssh-pubkey\"}}}"
I1030 02:07:28.437094 1 recorder.go:104] "Updated KubevirtCluster \"sveltos-member-02/sveltos-member-02-kwwcd\"" logger="events" type="Normal" object={"kind":"Cluster","namespace":"sveltos-member-02","name":"sveltos-member-02","uid":"641a4ac5-77b3-4e6d-9167-414fef24c1cd","apiVersion":"cluster.x-k8s.io/v1beta2","resourceVersion":"2529272"} reason="TopologyUpdate"
I1030 02:07:28.455076 1 reconcile_state.go:1238] "Patching KamajiControlPlane" controller="topology/cluster" controllerGroup="cluster.x-k8s.io" controllerKind="Cluster" Cluster="sveltos-member-02/sveltos-member-02" namespace="sveltos-member-02" name="sveltos-member-02" reconcileID="95b4a1c6-12ad-4bd2-81a6-bbbbbc207003" KamajiControlPlane="sveltos-member-02/sveltos-member-02-dcbv7" KamajiControlPlane="sveltos-member-02/sveltos-member-02-dcbv7" diff="{\"spec\":{\"apiServer\":{\"extraVolumeMounts\":[]},\"controllerManager\":{\"extraVolumeMounts\":[]},\"deployment\":{\"extraContainers\":[],\"extraInitContainers\":[],\"extraVolumes\":[]},\"scheduler\":{\"extraArgs\":[],\"extraVolumeMounts\":[]}}}"
I1030 02:07:28.468646 1 recorder.go:104] "Updated KamajiControlPlane \"sveltos-member-02/sveltos-member-02-dcbv7\"" logger="events" type="Normal" object={"kind":"Cluster","namespace":"sveltos-member-02","name":"sveltos-member-02","uid":"641a4ac5-77b3-4e6d-9167-414fef24c1cd","apiVersion":"cluster.x-k8s.io/v1beta2","resourceVersion":"2529272"} reason="TopologyUpdate"
I1030 02:07:28.532085 1 reconcile_state.go:1238] "Patching KubevirtCluster" controller="topology/cluster" controllerGroup="cluster.x-k8s.io" controllerKind="Cluster" Cluster="sveltos-member-02/sveltos-member-02" namespace="sveltos-member-02" name="sveltos-member-02" reconcileID="da94f8c2-3f41-401f-b6c4-794429d38769" KubevirtCluster="sveltos-member-02/sveltos-member-02-kwwcd" KubevirtCluster="sveltos-member-02/sveltos-member-02-kwwcd" diff="{\"spec\":{\"sshKeys\":{\"dataSecretName\":\"sveltos-member-02-ssh-pubkey\"}}}"
I1030 02:07:28.585153 1 recorder.go:104] "Updated KubevirtCluster \"sveltos-member-02/sveltos-member-02-kwwcd\"" logger="events" type="Normal" object={"kind":"Cluster","namespace":"sveltos-member-02","name":"sveltos-member-02","uid":"641a4ac5-77b3-4e6d-9167-414fef24c1cd","apiVersion":"cluster.x-k8s.io/v1beta2","resourceVersion":"2529272"} reason="TopologyUpdate"
What did you expect to happen?
nomal
Cluster API version
newest
Kubernetes version
1.30.2
Anything else you would like to add?
No response
Label(s) to be applied
/kind bug
One or more /area label. See https://github.com/kubernetes-sigs/cluster-api/labels?q=area for the list of labels.