Skip to content

Commit 545f635

Browse files
visheshtanksalevisheshtanksale
committed
Fixing HPA creation when multiple metrics specs are specified
Signed-off-by: Vishesh Tanksale <[email protected]>
1 parent e101973 commit 545f635

File tree

5 files changed

+129
-11
lines changed

5 files changed

+129
-11
lines changed

‎internal/controller/platform/standalone/nimservice.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
"github.com/go-logr/logr"
2929
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
3030
appsv1 "k8s.io/api/apps/v1"
31-
autoscalingv1 "k8s.io/api/autoscaling/v1"
31+
autoscalingv2 "k8s.io/api/autoscaling/v2"
3232
corev1 "k8s.io/api/core/v1"
3333
networkingv1 "k8s.io/api/networking/v1"
3434
rbacv1 "k8s.io/api/rbac/v1"
@@ -125,7 +125,7 @@ func (r *NIMServiceReconciler) reconcileNIMService(ctx context.Context, nimServi
125125

126126
// Sync HPA
127127
if nimService.IsAutoScalingEnabled() {
128-
err = r.renderAndSyncResource(ctx, nimService, &renderer, &autoscalingv1.HorizontalPodAutoscaler{}, func() (client.Object, error) {
128+
err = r.renderAndSyncResource(ctx, nimService, &renderer, &autoscalingv2.HorizontalPodAutoscaler{}, func() (client.Object, error) {
129129
return renderer.HPA(nimService.GetHPAParams())
130130
}, "hpa", conditions.ReasonHPAFailed)
131131
if err != nil {

‎internal/controller/platform/standalone/nimservice_test.go

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import (
3232
. "github.com/onsi/ginkgo/v2"
3333
. "github.com/onsi/gomega"
3434
appsv1 "k8s.io/api/apps/v1"
35-
autoscalingv1 "k8s.io/api/autoscaling/v1"
35+
autoscalingv2 "k8s.io/api/autoscaling/v2"
3636
corev1 "k8s.io/api/core/v1"
3737
networkingv1 "k8s.io/api/networking/v1"
3838
rbacv1 "k8s.io/api/rbac/v1"
@@ -79,7 +79,7 @@ var _ = Describe("NIMServiceReconciler for a standalone platform", func() {
7979
Expect(appsv1alpha1.AddToScheme(scheme)).To(Succeed())
8080
Expect(appsv1.AddToScheme(scheme)).To(Succeed())
8181
Expect(rbacv1.AddToScheme(scheme)).To(Succeed())
82-
Expect(autoscalingv1.AddToScheme(scheme)).To(Succeed())
82+
Expect(autoscalingv2.AddToScheme(scheme)).To(Succeed())
8383
Expect(networkingv1.AddToScheme(scheme)).To(Succeed())
8484
Expect(corev1.AddToScheme(scheme)).To(Succeed())
8585

@@ -99,6 +99,7 @@ var _ = Describe("NIMServiceReconciler for a standalone platform", func() {
9999
renderer: render.NewRenderer(path.Join(strings.TrimSuffix(cwd, "internal/controller/platform/standalone"), "manifests")),
100100
}
101101
pvcName := "test-pvc"
102+
minReplicas := int32(1)
102103
nimService = &appsv1alpha1.NIMService{
103104
ObjectMeta: metav1.ObjectMeta{
104105
Name: "test-nimservice",
@@ -172,7 +173,31 @@ var _ = Describe("NIMServiceReconciler for a standalone platform", func() {
172173
Effect: corev1.TaintEffectNoSchedule,
173174
},
174175
},
175-
Scale: appsv1alpha1.Autoscaling{Enabled: ptr.To[bool](false)},
176+
Scale: appsv1alpha1.Autoscaling{
177+
Enabled: ptr.To[bool](true),
178+
HPA: appsv1alpha1.HorizontalPodAutoscalerSpec{
179+
MinReplicas: &minReplicas,
180+
MaxReplicas: 10,
181+
Metrics: []autoscalingv2.MetricSpec{
182+
{
183+
Type: autoscalingv2.ResourceMetricSourceType,
184+
Resource: &autoscalingv2.ResourceMetricSource{
185+
Target: autoscalingv2.MetricTarget{
186+
Type: autoscalingv2.UtilizationMetricType,
187+
},
188+
},
189+
},
190+
{
191+
Type: autoscalingv2.PodsMetricSourceType,
192+
Pods: &autoscalingv2.PodsMetricSource{
193+
Target: autoscalingv2.MetricTarget{
194+
Type: autoscalingv2.UtilizationMetricType,
195+
},
196+
},
197+
},
198+
},
199+
},
200+
},
176201
ReadinessProbe: appsv1alpha1.Probe{
177202
Enabled: &boolTrue,
178203
Probe: &corev1.Probe{
@@ -323,6 +348,15 @@ var _ = Describe("NIMServiceReconciler for a standalone platform", func() {
323348
Expect(service.Name).To(Equal(nimService.GetName()))
324349
Expect(service.Namespace).To(Equal(nimService.GetNamespace()))
325350

351+
// HPA should be deployed
352+
hpa := &autoscalingv2.HorizontalPodAutoscaler{}
353+
err = client.Get(context.TODO(), namespacedName, hpa)
354+
Expect(err).NotTo(HaveOccurred())
355+
Expect(hpa.Name).To(Equal(nimService.GetName()))
356+
Expect(hpa.Namespace).To(Equal(nimService.GetNamespace()))
357+
Expect(*hpa.Spec.MinReplicas).To(Equal(int32(1)))
358+
Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10)))
359+
326360
// Deployment should be created
327361
deployment := &appsv1.Deployment{}
328362
err = client.Get(context.TODO(), namespacedName, deployment)

‎internal/render/render.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,8 @@ func (r *textTemplateRenderer) HPA(params *types.HPAParams) (*autoscalingv2.Hori
348348
if err != nil {
349349
return nil, fmt.Errorf("error converting unstructured object to HPA: %w", err)
350350
}
351+
// Sorting to avoid unnecessary object updates
352+
hpa.Spec.Metrics = utils.SortHPAMetricsSpec(hpa.Spec.Metrics)
351353
return hpa, nil
352354
}
353355

‎internal/render/render_test.go

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,5 +360,50 @@ var _ = Describe("K8s Resources Rendering", func() {
360360
Expect(*hpa.Spec.MinReplicas).To(Equal(int32(1)))
361361
Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10)))
362362
})
363+
364+
It("should render HPA template and sort metrics spec correctly", func() {
365+
minRep := int32(1)
366+
params := types.HPAParams{
367+
Enabled: true,
368+
Name: "test-hpa",
369+
Namespace: "default",
370+
HPASpec: autoscalingv2.HorizontalPodAutoscalerSpec{
371+
ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
372+
Name: "test-deployment",
373+
Kind: "Deployment",
374+
APIVersion: "apps/v1",
375+
},
376+
MinReplicas: &minRep,
377+
MaxReplicas: 10,
378+
Metrics: []autoscalingv2.MetricSpec{
379+
{
380+
Type: autoscalingv2.ResourceMetricSourceType,
381+
Resource: &autoscalingv2.ResourceMetricSource{
382+
Target: autoscalingv2.MetricTarget{
383+
Type: autoscalingv2.UtilizationMetricType,
384+
},
385+
},
386+
},
387+
{
388+
Type: autoscalingv2.PodsMetricSourceType,
389+
Pods: &autoscalingv2.PodsMetricSource{
390+
Target: autoscalingv2.MetricTarget{
391+
Type: autoscalingv2.UtilizationMetricType,
392+
},
393+
},
394+
},
395+
},
396+
},
397+
}
398+
r := render.NewRenderer(templatesDir)
399+
hpa, err := r.HPA(&params)
400+
Expect(err).NotTo(HaveOccurred())
401+
Expect(hpa.Name).To(Equal("test-hpa"))
402+
Expect(hpa.Namespace).To(Equal("default"))
403+
Expect(*hpa.Spec.MinReplicas).To(Equal(int32(1)))
404+
Expect(hpa.Spec.MaxReplicas).To(Equal(int32(10)))
405+
Expect(hpa.Spec.Metrics[0].Type).To(Equal(autoscalingv2.PodsMetricSourceType))
406+
Expect(hpa.Spec.Metrics[1].Type).To(Equal(autoscalingv2.ResourceMetricSourceType))
407+
})
363408
})
364409
})

‎internal/utils/utils.go

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,13 @@ import (
2323
"hash/fnv"
2424
"os"
2525
"path/filepath"
26+
"reflect"
2627
"sort"
2728
"strings"
2829

30+
autoscalingv2 "k8s.io/api/autoscaling/v2"
2931
corev1 "k8s.io/api/core/v1"
3032
"k8s.io/apimachinery/pkg/util/rand"
31-
"reflect"
3233
"sigs.k8s.io/controller-runtime/pkg/client"
3334
)
3435

@@ -131,13 +132,30 @@ func SortKeys(obj interface{}) interface{} {
131132
}
132133
return sortedMap
133134
case []interface{}:
134-
// Check if the slice contains maps and sort them if so
135+
// Check if the slice contains maps and sort them by the "name" field or the first available field
135136
if len(obj) > 0 {
137+
136138
if _, ok := obj[0].(map[string]interface{}); ok {
137-
sort.Slice(obj, func(i, j int) bool {
138-
iName := obj[i].(map[string]interface{})["name"].(string)
139-
jName := obj[j].(map[string]interface{})["name"].(string)
140-
return iName < jName
139+
sort.SliceStable(obj, func(i, j int) bool {
140+
iMap, iOk := obj[i].(map[string]interface{})
141+
jMap, jOk := obj[j].(map[string]interface{})
142+
if iOk && jOk {
143+
// Try to sort by "name" if present
144+
iName, iNameOk := iMap["name"].(string)
145+
jName, jNameOk := jMap["name"].(string)
146+
if iNameOk && jNameOk {
147+
return iName < jName
148+
}
149+
150+
// If "name" is not available, sort by the first key in each map
151+
if len(iMap) > 0 && len(jMap) > 0 {
152+
iFirstKey := firstKey(iMap)
153+
jFirstKey := firstKey(jMap)
154+
return iFirstKey < jFirstKey
155+
}
156+
}
157+
// If no valid comparison is possible, maintain the original order
158+
return false
141159
})
142160
}
143161
}
@@ -148,6 +166,16 @@ func SortKeys(obj interface{}) interface{} {
148166
return obj
149167
}
150168

169+
// Helper function to get the first key of a map (alphabetically sorted)
170+
func firstKey(m map[string]interface{}) string {
171+
keys := make([]string, 0, len(m))
172+
for k := range m {
173+
keys = append(keys, k)
174+
}
175+
sort.Strings(keys)
176+
return keys[0]
177+
}
178+
151179
// GetResourceHash returns a consistent hash for the given object spec
152180
func GetResourceHash(obj client.Object) string {
153181
// Convert obj to a map[string]interface{}
@@ -242,3 +270,12 @@ func IsEqual[T client.Object](existing, desired T, fieldsToCompare ...string) bo
242270

243271
return true
244272
}
273+
274+
func SortHPAMetricsSpec(metrics []autoscalingv2.MetricSpec) []autoscalingv2.MetricSpec {
275+
sort.Slice(metrics, func(i, j int) bool {
276+
iMetricsType := metrics[i].Type
277+
jMetricsType := metrics[j].Type
278+
return iMetricsType < jMetricsType
279+
})
280+
return metrics
281+
}

0 commit comments

Comments
 (0)