Skip to content

Commit

Permalink
Remobed PDB logic from ensureMismatchedPodsAreDeleted. Improved PDB l…
Browse files Browse the repository at this point in the history
…ogging. A few code refactors
  • Loading branch information
triceras committed Jan 28, 2025
1 parent 2d82ce6 commit 69a3348
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 70 deletions.
2 changes: 0 additions & 2 deletions api/v1alpha1/humiocluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -316,13 +316,11 @@ type HumioNodePoolSpec struct {
type HumioPodDisruptionBudgetSpec struct {
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Format=int-or-string
// +kubebuilder:validation:Immutable
// MinAvailable is the minimum number of pods that must be available during a disruption.
MinAvailable *intstr.IntOrString `json:"minAvailable,omitempty"`

// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Format=int-or-string
// +kubebuilder:validation:Immutable
// MaxUnavailable is the maximum number of pods that can be unavailable during a disruption.
MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`

Expand Down
75 changes: 7 additions & 68 deletions controllers/humiocluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -1928,64 +1928,6 @@ func (r *HumioClusterReconciler) ensureMismatchedPodsAreDeleted(ctx context.Cont

podsForDeletion := desiredLifecycleState.podsToBeReplaced

pdbSpec := hnp.GetPodDisruptionBudget()
r.Log.Info("Entering PDB enforcement check", "nodePool", hnp.GetNodePoolName(), "pdbSpec", pdbSpec, "pdbSpec.Enabled", pdbSpec != nil && pdbSpec.Enabled)
if pdbSpec != nil && pdbSpec.Enabled {
pdbName := fmt.Sprintf("%s-%s-pdb", hc.Name, hnp.GetNodePoolName())
pdb := &policyv1.PodDisruptionBudget{}

r.Log.Info("Fetching PDB",
"pdbName", pdbName,
"namespace", hnp.GetNamespace())

err := r.Get(ctx, types.NamespacedName{
Name: pdbName,
Namespace: hnp.GetNamespace(),
}, pdb)

if err != nil {
if k8serrors.IsNotFound(err) {
r.Log.Info("PDB not found for node pool, proceeding without PDB check",
"pdb", pdbName,
"namespace", hnp.GetNamespace(),
"nodePool", hnp.GetNodePoolName())
} else {
return reconcile.Result{}, fmt.Errorf("failed to get PDB for node pool %s: %w",
hnp.GetNodePoolName(), err)
}
} else {
disruptionsAllowed := pdb.Status.DisruptionsAllowed
podsToDeleteCount := len(podsForDeletion)

r.Log.Info("PDB Status",
"pdbName", pdbName,
"disruptionsAllowed", disruptionsAllowed,
"podsToDeleteCount", podsToDeleteCount)

if podsToDeleteCount > int(disruptionsAllowed) {
r.Log.Info("PDB preventing pod deletion",
"pdbName", pdbName,
"disruptionsAllowed", disruptionsAllowed,
"podsToDeleteCount", podsToDeleteCount)

return reconcile.Result{RequeueAfter: time.Minute}, fmt.Errorf(
"scale down blocked by PDB for node pool %s: disruptions allowed=%d, pods to delete=%d",
hnp.GetNodePoolName(),
disruptionsAllowed,
podsToDeleteCount)
}

r.Log.Info("PDB allows pod deletion",
"pdbName", pdbName,
"disruptionsAllowed", disruptionsAllowed,
"podsToDeleteCount", podsToDeleteCount)
}
} else {
r.Log.Info("PDB enforcement NOT ENABLED or no PDB spec",
"nodePool", hnp.GetNodePoolName(),
"pdbSpec", pdbSpec)
}

// if zone awareness is enabled, we pin a zone until we're done replacing all pods in that zone,
// this is repeated for each zone with pods that needs replacing
if *hnp.GetUpdateStrategy().EnableZoneAwareness && !helpers.UseEnvtest() {
Expand Down Expand Up @@ -2346,11 +2288,6 @@ func (r *HumioClusterReconciler) cleanupUnusedResources(ctx context.Context, hc
}
}

if err := r.cleanupOrphanedPDBs(ctx, hc, &humioNodePools); err != nil {
return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions().
withMessage(err.Error()))
}

return reconcile.Result{}, nil
}

Expand Down Expand Up @@ -2448,7 +2385,7 @@ func (r *HumioClusterReconciler) reconcileSinglePDB(ctx context.Context, hc *hum

desiredPDB, err := r.constructPDB(hc, hnp, pdbSpec)
if err != nil {
r.Log.Error(err, "failed to construct PDB", "pdbName", hnp.GetPodDisruptionBudgetName(), "namespace", hnp.GetNamespace())
r.Log.Error(err, "failed to construct PDB", "pdbName", hnp.GetPodDisruptionBudgetName())
return fmt.Errorf("failed to construct PDB: %w", err)
}

Expand All @@ -2475,17 +2412,19 @@ func (r *HumioClusterReconciler) constructPDB(hc *humiov1alpha1.HumioCluster, hn
ObjectMeta: metav1.ObjectMeta{
Name: pdbName,
Namespace: hc.Namespace,
Labels: kubernetes.LabelsForHumio(hc.Name), // Add node pool name label if needed
Labels: kubernetes.LabelsForHumio(hc.Name),
Finalizers: []string{HumioProtectionFinalizer},
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(hc, humiov1alpha1.GroupVersion.WithKind("HumioCluster")),
},
},
Spec: policyv1.PodDisruptionBudgetSpec{
Selector: selector,
},
}

// Set controller reference using controller-runtime utility
if err := controllerutil.SetControllerReference(hc, pdb, r.Scheme()); err != nil {
return nil, fmt.Errorf("failed to set controller reference: %w", err)
}

if minAvailable != nil {
pdb.Spec.MinAvailable = minAvailable
} else {
Expand Down

0 comments on commit 69a3348

Please sign in to comment.