Skip to content

Commit 75e6587

Browse files
oliviassssoliviassss
committed
add flag to specify resource prefix for resource tags
1 parent 990ee2e commit 75e6587

11 files changed

+209
-68
lines changed

‎controllers/ingress/group_controller.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,7 @@ import (
3434
)
3535

3636
const (
37-
ingressTagPrefix = "ingress.k8s.aws"
38-
controllerName = "ingress"
37+
controllerName = "ingress"
3938

4039
// the groupVersion of used Ingress & IngressClass resource.
4140
ingressResourcesGroupVersion = "networking.k8s.io/v1"
@@ -53,7 +52,7 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder
5352
authConfigBuilder := ingress.NewDefaultAuthConfigBuilder(annotationParser)
5453
enhancedBackendBuilder := ingress.NewDefaultEnhancedBackendBuilder(k8sClient, annotationParser, authConfigBuilder, controllerConfig.IngressConfig.TolerateNonExistentBackendService, controllerConfig.IngressConfig.TolerateNonExistentBackendAction)
5554
referenceIndexer := ingress.NewDefaultReferenceIndexer(enhancedBackendBuilder, authConfigBuilder, logger)
56-
trackingProvider := tracking.NewDefaultProvider(ingressTagPrefix, controllerConfig.ClusterName)
55+
trackingProvider := tracking.NewDefaultProvider(controllerConfig.ResourcePrefix[config.ClusterTagPrefixKey], controllerConfig.ResourcePrefix[config.IngressTagPrefixKey], controllerConfig.ClusterName)
5756
modelBuilder := ingress.NewDefaultModelBuilder(k8sClient, eventRecorder,
5857
cloud.EC2(), cloud.ELBV2(), cloud.ACM(),
5958
annotationParser, subnetsResolver,
@@ -63,7 +62,7 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder
6362
controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger)
6463
stackMarshaller := deploy.NewDefaultStackMarshaller()
6564
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager,
66-
controllerConfig, ingressTagPrefix, logger)
65+
controllerConfig, controllerConfig.ResourcePrefix[config.ClusterTagPrefixKey], controllerConfig.ResourcePrefix[config.IngressTagPrefixKey], logger)
6766
classLoader := ingress.NewDefaultClassLoader(k8sClient, true)
6867
classAnnotationMatcher := ingress.NewDefaultClassAnnotationMatcher(controllerConfig.IngressConfig.IngressClass)
6968
manageIngressesWithoutIngressClass := controllerConfig.IngressConfig.IngressClass == ""

‎controllers/service/service_controller.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ import (
2929

3030
const (
3131
serviceFinalizer = "service.k8s.aws/resources"
32-
serviceTagPrefix = "service.k8s.aws"
3332
serviceAnnotationPrefix = "service.beta.kubernetes.io"
3433
controllerName = "service"
3534
)
@@ -41,14 +40,14 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde
4140
backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger) *serviceReconciler {
4241

4342
annotationParser := annotations.NewSuffixAnnotationParser(serviceAnnotationPrefix)
44-
trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName)
43+
trackingProvider := tracking.NewDefaultProvider(controllerConfig.ResourcePrefix[config.ClusterTagPrefixKey], controllerConfig.ResourcePrefix[config.ServiceTagPrefixKey], controllerConfig.ClusterName)
4544
serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
4645
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
4746
elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
4847
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
4948
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger)
5049
stackMarshaller := deploy.NewDefaultStackMarshaller()
51-
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger)
50+
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, controllerConfig.ResourcePrefix[config.ClusterTagPrefixKey], controllerConfig.ResourcePrefix[config.ServiceTagPrefixKey], logger)
5251
return &serviceReconciler{
5352
k8sClient: k8sClient,
5453
eventRecorder: eventRecorder,

‎pkg/config/controller_config.go

Lines changed: 83 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package config
22

33
import (
4+
"fmt"
45
"strings"
56
"time"
67

@@ -16,6 +17,7 @@ const (
1617
flagLogLevel = "log-level"
1718
flagK8sClusterName = "cluster-name"
1819
flagDefaultTags = "default-tags"
20+
flagResourcePrefix = "resource-prefix"
1921
flagDefaultTargetType = "default-target-type"
2022
flagExternalManagedTags = "external-managed-tags"
2123
flagServiceTargetENISGTags = "service-target-eni-security-group-tags"
@@ -27,24 +29,42 @@ const (
2729
flagBackendSecurityGroup = "backend-security-group"
2830
flagEnableEndpointSlices = "enable-endpoint-slices"
2931
flagDisableRestrictedSGRules = "disable-restricted-sg-rules"
30-
defaultLogLevel = "info"
31-
defaultMaxConcurrentReconciles = 3
32-
defaultMaxExponentialBackoffDelay = time.Second * 1000
33-
defaultSSLPolicy = "ELBSecurityPolicy-2016-08"
34-
defaultEnableBackendSG = true
35-
defaultEnableEndpointSlices = false
36-
defaultDisableRestrictedSGRules = false
32+
33+
ClusterTagPrefixKey = "clusterTagPrefix"
34+
IngressTagPrefixKey = "ingressTagPrefix"
35+
ServiceTagPrefixKey = "serviceTagPrefix"
36+
BackendSGNamePrefixKey = "backendSGNamePrefix"
37+
ClusterSgRuleLabelPrefixKey = "clusterSgRuleLabelPrefix"
38+
39+
defaultClusterTagPrefix = "elbv2.k8s.aws"
40+
defaultIngressTagPrefix = "ingress.k8s.aws"
41+
defaultServiceTagPrefix = "service.k8s.aws"
42+
defaultBackendSGNamePrefix = "k8s-traffic"
43+
defaultClusterSgRuleLabelPrefix = "elbv2.k8s.aws"
44+
defaultLogLevel = "info"
45+
defaultMaxConcurrentReconciles = 3
46+
defaultMaxExponentialBackoffDelay = time.Second * 1000
47+
defaultSSLPolicy = "ELBSecurityPolicy-2016-08"
48+
defaultEnableBackendSG = true
49+
defaultEnableEndpointSlices = false
50+
defaultDisableRestrictedSGRules = false
3751
)
3852

3953
var (
40-
trackingTagKeys = sets.NewString(
41-
"elbv2.k8s.aws/cluster",
42-
"elbv2.k8s.aws/resource",
43-
"ingress.k8s.aws/stack",
44-
"ingress.k8s.aws/resource",
45-
"service.k8s.aws/stack",
46-
"service.k8s.aws/resource",
54+
validPrefixKeys = sets.NewString(
55+
ClusterTagPrefixKey,
56+
IngressTagPrefixKey,
57+
ServiceTagPrefixKey,
58+
BackendSGNamePrefixKey,
59+
ClusterSgRuleLabelPrefixKey,
4760
)
61+
defaultResourcePrefix = map[string]string{
62+
ClusterTagPrefixKey: defaultClusterTagPrefix,
63+
IngressTagPrefixKey: defaultIngressTagPrefix,
64+
ServiceTagPrefixKey: defaultServiceTagPrefix,
65+
BackendSGNamePrefixKey: defaultBackendSGNamePrefix,
66+
ClusterSgRuleLabelPrefixKey: defaultClusterSgRuleLabelPrefix,
67+
}
4868
)
4969

5070
// ControllerConfig contains the controller configuration
@@ -69,6 +89,9 @@ type ControllerConfig struct {
6989
// Default AWS Tags that will be applied to all AWS resources managed by this controller.
7090
DefaultTags map[string]string
7191

92+
// ResourcePrefix provides prefix for resource tags, backend SG name and worker node SG rules label.
93+
ResourcePrefix map[string]string
94+
7295
// Default target type for Ingress and Service objects
7396
DefaultTargetType string
7497

@@ -134,10 +157,13 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
134157
"Disable the usage of restricted security group rules")
135158
fs.StringToStringVar(&cfg.ServiceTargetENISGTags, flagServiceTargetENISGTags, nil,
136159
"AWS Tags, in addition to cluster tags, for finding the target ENI security group to which to add inbound rules from NLBs")
160+
fs.StringToStringVar(&cfg.ResourcePrefix, flagResourcePrefix, defaultResourcePrefix,
161+
"the prefixes for resource tags, backend SG name and worker node SG rules label.")
162+
163+
cfg.mergeDefaultResourcePrefixVal()
137164
cfg.FeatureGates.BindFlags(fs)
138165
cfg.AWSConfig.BindFlags(fs)
139166
cfg.RuntimeConfig.BindFlags(fs)
140-
141167
cfg.PodWebhookConfig.BindFlags(fs)
142168
cfg.IngressConfig.BindFlags(fs)
143169
cfg.AddonsConfig.BindFlags(fs)
@@ -150,10 +176,23 @@ func (cfg *ControllerConfig) Validate() error {
150176
return errors.New("kubernetes cluster name must be specified")
151177
}
152178

153-
if err := cfg.validateDefaultTagsCollisionWithTrackingTags(); err != nil {
179+
if err := cfg.validateResourcePrefixKeys(); err != nil {
180+
return err
181+
}
182+
183+
trackingTagKeys := sets.New[string](
184+
cfg.ResourcePrefix[ClusterTagPrefixKey]+"/cluster",
185+
cfg.ResourcePrefix[ClusterTagPrefixKey]+"/resource",
186+
cfg.ResourcePrefix[IngressTagPrefixKey]+"/stack",
187+
cfg.ResourcePrefix[IngressTagPrefixKey]+"/resource",
188+
cfg.ResourcePrefix[ServiceTagPrefixKey]+"/stack",
189+
cfg.ResourcePrefix[ServiceTagPrefixKey]+"/resource",
190+
)
191+
192+
if err := cfg.validateDefaultTagsCollisionWithTrackingTags(trackingTagKeys); err != nil {
154193
return err
155194
}
156-
if err := cfg.validateExternalManagedTagsCollisionWithTrackingTags(); err != nil {
195+
if err := cfg.validateExternalManagedTagsCollisionWithTrackingTags(trackingTagKeys); err != nil {
157196
return err
158197
}
159198
if err := cfg.validateExternalManagedTagsCollisionWithDefaultTags(); err != nil {
@@ -168,7 +207,7 @@ func (cfg *ControllerConfig) Validate() error {
168207
return nil
169208
}
170209

171-
func (cfg *ControllerConfig) validateDefaultTagsCollisionWithTrackingTags() error {
210+
func (cfg *ControllerConfig) validateDefaultTagsCollisionWithTrackingTags(trackingTagKeys sets.Set[string]) error {
172211
for tagKey := range cfg.DefaultTags {
173212
if trackingTagKeys.Has(tagKey) {
174213
return errors.Errorf("tag key %v cannot be specified in %v flag", tagKey, flagDefaultTags)
@@ -177,7 +216,7 @@ func (cfg *ControllerConfig) validateDefaultTagsCollisionWithTrackingTags() erro
177216
return nil
178217
}
179218

180-
func (cfg *ControllerConfig) validateExternalManagedTagsCollisionWithTrackingTags() error {
219+
func (cfg *ControllerConfig) validateExternalManagedTagsCollisionWithTrackingTags(trackingTagKeys sets.Set[string]) error {
181220
for _, tagKey := range cfg.ExternalManagedTags {
182221
if trackingTagKeys.Has(tagKey) {
183222
return errors.Errorf("tag key %v cannot be specified in %v flag", tagKey, flagExternalManagedTags)
@@ -214,3 +253,28 @@ func (cfg *ControllerConfig) validateBackendSecurityGroupConfiguration() error {
214253
}
215254
return nil
216255
}
256+
257+
func (cfg *ControllerConfig) validateResourcePrefixKeys() error {
258+
keys := make([]string, 0, len(cfg.ResourcePrefix))
259+
for key := range cfg.ResourcePrefix {
260+
if !validPrefixKeys.Has(key) {
261+
return fmt.Errorf("invalid key: %s. Valid keys are: %v", key, validPrefixKeys.List())
262+
}
263+
keys = append(keys, key)
264+
}
265+
if len(keys) != len(validPrefixKeys.List()) {
266+
return fmt.Errorf("invalid number of keys. Expected %d keys, but got %d keys",
267+
len(validPrefixKeys.List()), len(keys))
268+
}
269+
return nil
270+
}
271+
272+
// mergeDefaultResourcePrefixVal make sure the ResourcePrefix map always has default val for unspecified key in user-passed flag
273+
func (cfg *ControllerConfig) mergeDefaultResourcePrefixVal() {
274+
// Merge user-provided values with defaults
275+
for key, defaultVal := range defaultResourcePrefix {
276+
if _, exists := cfg.ResourcePrefix[key]; !exists {
277+
cfg.ResourcePrefix[key] = defaultVal
278+
}
279+
}
280+
}

‎pkg/config/controller_config_test.go

Lines changed: 82 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package config
33
import (
44
"github.com/pkg/errors"
55
"github.com/stretchr/testify/assert"
6+
"k8s.io/apimachinery/pkg/util/sets"
67
"testing"
78
)
89

@@ -46,7 +47,15 @@ func TestControllerConfig_validateDefaultTagsCollisionWithTrackingTags(t *testin
4647
cfg := &ControllerConfig{
4748
DefaultTags: tt.fields.DefaultTags,
4849
}
49-
err := cfg.validateDefaultTagsCollisionWithTrackingTags()
50+
trackingTagKeys := sets.New[string](
51+
"elbv2.k8s.aws/cluster",
52+
"elbv2.k8s.aws/resource",
53+
"ingress.k8s.aws/stack",
54+
"ingress.k8s.aws/resource",
55+
"service.k8s.aws/stack",
56+
"service.k8s.aws/resource",
57+
)
58+
err := cfg.validateDefaultTagsCollisionWithTrackingTags(trackingTagKeys)
5059
if tt.wantErr != nil {
5160
assert.EqualError(t, err, tt.wantErr.Error())
5261
} else {
@@ -92,7 +101,15 @@ func TestControllerConfig_validateExternalManagedTagsCollisionWithTrackingTags(t
92101
cfg := &ControllerConfig{
93102
ExternalManagedTags: tt.fields.ExternalManagedTags,
94103
}
95-
err := cfg.validateExternalManagedTagsCollisionWithTrackingTags()
104+
trackingTagKeys := sets.New[string](
105+
"elbv2.k8s.aws/cluster",
106+
"elbv2.k8s.aws/resource",
107+
"ingress.k8s.aws/stack",
108+
"ingress.k8s.aws/resource",
109+
"service.k8s.aws/stack",
110+
"service.k8s.aws/resource",
111+
)
112+
err := cfg.validateExternalManagedTagsCollisionWithTrackingTags(trackingTagKeys)
96113
if tt.wantErr != nil {
97114
assert.EqualError(t, err, tt.wantErr.Error())
98115
} else {
@@ -164,3 +181,66 @@ func TestControllerConfig_validateExternalManagedTagsCollisionWithDefaultTags(t
164181
})
165182
}
166183
}
184+
185+
func TestControllerConfig_validateResourcePrefixKeys(t *testing.T) {
186+
type fields struct {
187+
ResourcePrefix map[string]string
188+
}
189+
tests := []struct {
190+
name string
191+
fields fields
192+
wantErr error
193+
}{
194+
{
195+
name: "resource prefix has all keys",
196+
fields: fields{
197+
ResourcePrefix: map[string]string{
198+
"clusterTagPrefix": "elbv2.k8s.aws",
199+
"ingressTagPrefix": "ingress.k8s.aws",
200+
"serviceTagPrefix": "service.k8s.aws",
201+
"backendSGNamePrefix": "k8s-traffic",
202+
"clusterSgRuleLabelPrefix": "elbv2.k8s.aws",
203+
},
204+
},
205+
wantErr: nil,
206+
},
207+
{
208+
name: "resource prefix has some invalid keys",
209+
fields: fields{
210+
ResourcePrefix: map[string]string{
211+
"clusterTagPrefix": "elbv2.k8s.aws",
212+
"ingressTagPrefix": "ingress.k8s.aws",
213+
"serviceTagPrefix": "service.k8s.aws",
214+
"backendSGNamePrefix": "k8s-traffic",
215+
"myKey": "myVal",
216+
},
217+
},
218+
wantErr: errors.New("invalid key: myKey. Valid keys are: [backendSGNamePrefix clusterSgRuleLabelPrefix clusterTagPrefix ingressTagPrefix serviceTagPrefix]"),
219+
},
220+
{
221+
name: "resource prefix is missing some valid keys",
222+
fields: fields{
223+
ResourcePrefix: map[string]string{
224+
"clusterTagPrefix": "elbv2.k8s.aws",
225+
"ingressTagPrefix": "ingress.k8s.aws",
226+
"serviceTagPrefix": "service.k8s.aws",
227+
"backendSGNamePrefix": "k8s-traffic",
228+
},
229+
},
230+
wantErr: errors.New("invalid number of keys. Expected 5 keys, but got 4 keys"),
231+
},
232+
}
233+
for _, tt := range tests {
234+
t.Run(tt.name, func(t *testing.T) {
235+
cfg := &ControllerConfig{
236+
ResourcePrefix: tt.fields.ResourcePrefix,
237+
}
238+
err := cfg.validateResourcePrefixKeys()
239+
if tt.wantErr != nil {
240+
assert.EqualError(t, err, tt.wantErr.Error())
241+
} else {
242+
assert.NoError(t, err)
243+
}
244+
})
245+
}
246+
}

‎pkg/deploy/stack_deployer.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ type StackDeployer interface {
2626
func NewDefaultStackDeployer(cloud aws.Cloud, k8sClient client.Client,
2727
networkingSGManager networking.SecurityGroupManager, networkingSGReconciler networking.SecurityGroupReconciler,
2828
elbv2TaggingManager elbv2.TaggingManager,
29-
config config.ControllerConfig, tagPrefix string, logger logr.Logger) *defaultStackDeployer {
29+
config config.ControllerConfig, clusterTagPrefix string, resourceTagPrefix string, logger logr.Logger) *defaultStackDeployer {
3030

31-
trackingProvider := tracking.NewDefaultProvider(tagPrefix, config.ClusterName)
31+
trackingProvider := tracking.NewDefaultProvider(clusterTagPrefix, resourceTagPrefix, config.ClusterName)
3232
ec2TaggingManager := ec2.NewDefaultTaggingManager(cloud.EC2(), networkingSGManager, cloud.VpcID(), logger)
3333

3434
return &defaultStackDeployer{

0 commit comments

Comments
 (0)