Skip to content

Commit b488ca7

Browse files
gambol99gambol99
authored
[FEATURE] - Executor Resource Requests & Limits (#1501)
* [FEAT] - Executor Resource Requests & Limits Currently consumers are not permitted to set the executor resource requests/limits - this has created an error in the past, due to higher than expected requirements, hence it was bumped to 1Gi. For certain modules though this may not be enough, hence the feature to support controling the fields from the controller arguments * feat: ensure the inputs are valid quantities * fix: adjusting due to formatting issues * feat: removing the limitation by default and placing the decision on the administrator as to limits
1 parent d94143d commit b488ca7

File tree

8 files changed

+124
-54
lines changed

8 files changed

+124
-54
lines changed

‎cmd/controller/main.go

+4
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@ func main() {
7575
flags.IntVar(&config.MetricsPort, "metrics-port", 9090, "The port the metric endpoint binds to")
7676
flags.IntVar(&config.WebhookPort, "webhooks-port", 10081, "The port the webhook endpoint binds to")
7777
flags.StringSliceVar(&config.ExecutorSecrets, "executor-secret", []string{}, "Name of a secret in controller namespace which should be added to the job")
78+
flags.StringVar(&config.ExecutorMemoryRequest, "executor-memory-request", "32Mi", "The default memory request for the executor container")
79+
flags.StringVar(&config.ExecutorMemoryLimit, "executor-memory-limit", "", "The default memory limit for the executor container (default is no limit)")
80+
flags.StringVar(&config.ExecutorCPURequest, "executor-cpu-request", "5m", "The default CPU request for the executor container")
81+
flags.StringVar(&config.ExecutorCPULimit, "executor-cpu-limit", "", "The default CPU limit for the executor container (default is no limit)")
7882
flags.StringVar(&config.BackendTemplate, "backend-template", "", "Name of secret in the controller namespace containing a template for the terraform state")
7983
flags.StringVar(&config.ExecutorImage, "executor-image", fmt.Sprintf("ghcr.io/appvia/terranetes-executor:%s", version.Version), "The image to use for the executor")
8084
flags.StringVar(&config.InfracostsImage, "infracost-image", "infracosts/infracost:latest", "The image to use for the infracosts")

‎pkg/assets/job.yaml.tpl

+10-4
Original file line numberDiff line numberDiff line change
@@ -259,12 +259,18 @@ spec:
259259
optional: false
260260
{{- end }}
261261
resources:
262+
{{- if or (ne .DefaultExecutorCPULimit "") (ne .DefaultExecutorMemoryLimit "") }}
262263
limits:
263-
cpu: 1
264-
memory: 1Gi
264+
{{- if ne .DefaultExecutorCPULimit "" }}
265+
cpu: {{ .DefaultExecutorCPULimit }}
266+
{{- end }}
267+
{{- if ne .DefaultExecutorMemoryLimit "" }}
268+
memory: {{ .DefaultExecutorMemoryLimit }}
269+
{{- end }}
270+
{{- end }}
265271
requests:
266-
cpu: 5m
267-
memory: 32Mi
272+
cpu: {{ .DefaultExecutorCPURequest }}
273+
memory: {{ .DefaultExecutorMemoryRequest }}
268274
securityContext:
269275
capabilities:
270276
drop: [ALL]

‎pkg/controller/configuration/controller.go

+24
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
log "github.com/sirupsen/logrus"
2828
batchv1 "k8s.io/api/batch/v1"
2929
v1 "k8s.io/api/core/v1"
30+
"k8s.io/apimachinery/pkg/api/resource"
3031
"k8s.io/client-go/informers"
3132
"k8s.io/client-go/kubernetes"
3233
cache "k8s.io/client-go/tools/cache"
@@ -89,6 +90,14 @@ type Controller struct {
8990
// executors job everytime - these are configured by the platform team on the
9091
// cli options
9192
ExecutorSecrets []string
93+
// DefaultExecutorMemoryRequest is the default memory request for the executor container
94+
DefaultExecutorMemoryRequest string
95+
// DefaultExecutorMemoryLimit is the default memory limit for the executor container
96+
DefaultExecutorMemoryLimit string
97+
// DefaultExecutorCPURequest is the default CPU request for the executor container
98+
DefaultExecutorCPURequest string
99+
// DefaultExecutorCPULimit is the default CPU limit for the executor container
100+
DefaultExecutorCPULimit string
92101
// InfracostsImage is the image to use for all infracost jobs
93102
InfracostsImage string
94103
// InfracostsSecretName is the name of the secret containing the api and token
@@ -161,6 +170,21 @@ func (c *Controller) Add(mgr manager.Manager) error {
161170
"terraform_image": c.TerraformImage,
162171
}).Info("adding the configuration controller")
163172

173+
// @step: ensure the resource limits are valid
174+
for _, c := range []string{
175+
c.DefaultExecutorCPULimit,
176+
c.DefaultExecutorCPURequest,
177+
c.DefaultExecutorMemoryLimit,
178+
c.DefaultExecutorMemoryRequest,
179+
} {
180+
if c == "" {
181+
continue
182+
}
183+
if _, err := resource.ParseQuantity(c); err != nil {
184+
return fmt.Errorf("invalid resource quantity: %q, error: %w", c, err)
185+
}
186+
}
187+
164188
switch {
165189
case c.ControllerNamespace == "":
166190
return errors.New("job namespace is required")

‎pkg/controller/configuration/ensure.go

+16-12
Original file line numberDiff line numberDiff line change
@@ -815,18 +815,22 @@ func (c *Controller) ensureTerraformPlan(configuration *terraformv1alpha1.Config
815815
terraformv1alpha1.RetryAnnotation: configuration.GetAnnotations()[terraformv1alpha1.RetryAnnotation],
816816
terraformv1alpha1.JobTemplateHashLabel: state.jobTemplateHash,
817817
}),
818-
BackoffLimit: c.BackoffLimit,
819-
EnableInfraCosts: c.EnableInfracosts,
820-
ExecutorImage: c.ExecutorImage,
821-
ExecutorSecrets: c.ExecutorSecrets,
822-
InfracostsImage: c.InfracostsImage,
823-
InfracostsSecret: c.InfracostsSecretName,
824-
Namespace: c.ControllerNamespace,
825-
PolicyConstraint: state.checkovConstraint,
826-
PolicyImage: c.PolicyImage,
827-
SaveTerraformState: saveState,
828-
Template: state.jobTemplate,
829-
TerraformImage: GetTerraformImage(configuration, c.TerraformImage),
818+
BackoffLimit: c.BackoffLimit,
819+
DefaultExecutorCPULimit: c.DefaultExecutorCPULimit,
820+
DefaultExecutorCPURequest: c.DefaultExecutorCPURequest,
821+
DefaultExecutorMemoryLimit: c.DefaultExecutorMemoryLimit,
822+
DefaultExecutorMemoryRequest: c.DefaultExecutorMemoryRequest,
823+
EnableInfraCosts: c.EnableInfracosts,
824+
ExecutorImage: c.ExecutorImage,
825+
ExecutorSecrets: c.ExecutorSecrets,
826+
InfracostsImage: c.InfracostsImage,
827+
InfracostsSecret: c.InfracostsSecretName,
828+
Namespace: c.ControllerNamespace,
829+
PolicyConstraint: state.checkovConstraint,
830+
PolicyImage: c.PolicyImage,
831+
SaveTerraformState: saveState,
832+
Template: state.jobTemplate,
833+
TerraformImage: GetTerraformImage(configuration, c.TerraformImage),
830834
}
831835

832836
// @step: use the options to generate the job

‎pkg/controller/configuration/reconcile_test.go

+31-23
Original file line numberDiff line numberDiff line change
@@ -56,18 +56,22 @@ func TestController(t *testing.T) {
5656
func makeFakeController(cc client.Client) *Controller {
5757
recorder := &controllertests.FakeRecorder{}
5858
ctrl := &Controller{
59-
cc: cc,
60-
kc: kfake.NewSimpleClientset(),
61-
cache: cache.New(5*time.Minute, 10*time.Minute),
62-
recorder: recorder,
63-
BackoffLimit: 2,
64-
EnableInfracosts: false,
65-
EnableWatchers: true,
66-
ExecutorImage: "ghcr.io/appvia/terranetes-executor",
67-
InfracostsImage: "infracosts/infracost:latest",
68-
ControllerNamespace: "terraform-system",
69-
PolicyImage: "bridgecrew/checkov:2.0.1140",
70-
TerraformImage: "hashicorp/terraform:1.1.9",
59+
cc: cc,
60+
kc: kfake.NewSimpleClientset(),
61+
cache: cache.New(5*time.Minute, 10*time.Minute),
62+
recorder: recorder,
63+
BackoffLimit: 2,
64+
ControllerNamespace: "terraform-system",
65+
DefaultExecutorCPULimit: "1",
66+
DefaultExecutorCPURequest: "5m",
67+
DefaultExecutorMemoryLimit: "1Gi",
68+
DefaultExecutorMemoryRequest: "32Mi",
69+
EnableInfracosts: false,
70+
EnableWatchers: true,
71+
ExecutorImage: "ghcr.io/appvia/terranetes-executor",
72+
InfracostsImage: "infracosts/infracost:latest",
73+
PolicyImage: "bridgecrew/checkov:2.0.1140",
74+
TerraformImage: "hashicorp/terraform:1.1.9",
7175
}
7276

7377
return ctrl
@@ -928,17 +932,21 @@ var _ = Describe("Configuration Controller", func() {
928932

929933
recorder = &controllertests.FakeRecorder{}
930934
ctrl = &Controller{
931-
cc: cc,
932-
kc: kfake.NewSimpleClientset(),
933-
cache: cache.New(5*time.Minute, 10*time.Minute),
934-
recorder: recorder,
935-
EnableInfracosts: false,
936-
EnableWatchers: true,
937-
ExecutorImage: "ghcr.io/appvia/terranetes-executor",
938-
InfracostsImage: "infracosts/infracost:latest",
939-
ControllerNamespace: "default",
940-
PolicyImage: "bridgecrew/checkov:2.0.1140",
941-
TerraformImage: "hashicorp/terraform:1.1.9",
935+
cc: cc,
936+
kc: kfake.NewSimpleClientset(),
937+
cache: cache.New(5*time.Minute, 10*time.Minute),
938+
recorder: recorder,
939+
DefaultExecutorCPULimit: "1",
940+
DefaultExecutorCPURequest: "5m",
941+
DefaultExecutorMemoryLimit: "1Gi",
942+
DefaultExecutorMemoryRequest: "32Mi",
943+
EnableInfracosts: false,
944+
EnableWatchers: true,
945+
ExecutorImage: "ghcr.io/appvia/terranetes-executor",
946+
InfracostsImage: "infracosts/infracost:latest",
947+
ControllerNamespace: "default",
948+
PolicyImage: "bridgecrew/checkov:2.0.1140",
949+
TerraformImage: "hashicorp/terraform:1.1.9",
942950
}
943951
ctrl.cache.SetDefault(cfgNamespace, fixtures.NewNamespace(cfgNamespace))
944952
}

‎pkg/server/server.go

+19-15
Original file line numberDiff line numberDiff line change
@@ -171,21 +171,25 @@ func New(cfg *rest.Config, config Config) (*Server, error) {
171171

172172
// @step: ensure the configuration controller is enabled
173173
if err := (&configuration.Controller{
174-
BackendTemplate: config.BackendTemplate,
175-
BackoffLimit: config.BackoffLimit,
176-
ControllerJobLabels: jobLabels,
177-
ControllerNamespace: config.Namespace,
178-
EnableInfracosts: (config.InfracostsSecretName != ""),
179-
EnableTerraformVersions: config.EnableTerraformVersions,
180-
EnableWatchers: config.EnableWatchers,
181-
EnableWebhooks: config.EnableWebhooks,
182-
ExecutorImage: config.ExecutorImage,
183-
ExecutorSecrets: config.ExecutorSecrets,
184-
InfracostsImage: config.InfracostsImage,
185-
InfracostsSecretName: config.InfracostsSecretName,
186-
JobTemplate: config.JobTemplate,
187-
PolicyImage: config.PolicyImage,
188-
TerraformImage: config.TerraformImage,
174+
BackendTemplate: config.BackendTemplate,
175+
BackoffLimit: config.BackoffLimit,
176+
ControllerJobLabels: jobLabels,
177+
ControllerNamespace: config.Namespace,
178+
DefaultExecutorCPULimit: config.ExecutorCPULimit,
179+
DefaultExecutorCPURequest: config.ExecutorCPURequest,
180+
DefaultExecutorMemoryLimit: config.ExecutorMemoryLimit,
181+
DefaultExecutorMemoryRequest: config.ExecutorMemoryRequest,
182+
EnableInfracosts: (config.InfracostsSecretName != ""),
183+
EnableTerraformVersions: config.EnableTerraformVersions,
184+
EnableWatchers: config.EnableWatchers,
185+
EnableWebhooks: config.EnableWebhooks,
186+
ExecutorImage: config.ExecutorImage,
187+
ExecutorSecrets: config.ExecutorSecrets,
188+
InfracostsImage: config.InfracostsImage,
189+
InfracostsSecretName: config.InfracostsSecretName,
190+
JobTemplate: config.JobTemplate,
191+
PolicyImage: config.PolicyImage,
192+
TerraformImage: config.TerraformImage,
189193
}).Add(mgr); err != nil {
190194
return nil, fmt.Errorf("failed to create the configuration controller, error: %w", err)
191195
}

‎pkg/server/types.go

+8
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,14 @@ type Config struct {
5959
EnableWebhookPrefix bool
6060
// ExecutorImage is the image to use for the executor
6161
ExecutorImage string
62+
// ExecutorMemoryRequest is the memory request for the executor container (e.g. 1Gi, 2Gi)
63+
ExecutorMemoryRequest string
64+
// ExecutorMemoryLimit is the memory limit for the executor container (e.g. 1Gi, 2Gi)
65+
ExecutorMemoryLimit string
66+
// ExecutorCPURequest is the CPU request for the executor container (e.g. 1, 2)
67+
ExecutorCPURequest string
68+
// ExecutorCPULimit is the CPU limit for the executor container (e.g. 1, 2)
69+
ExecutorCPULimit string
6270
// ExecutorSecrets is a list of additional secrets to be added to the executor
6371
ExecutorSecrets []string
6472
// InfracostsSecretName is the name of the secret that contains the cost token and endpoint

‎pkg/utils/jobs/jobs.go

+12
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,14 @@ type Options struct {
5353
// BackoffLimit is the number of times we are willing to allow a job to fail
5454
// before we give up
5555
BackoffLimit int
56+
// DefaultExecutorMemoryRequest is the default memory request for the executor
57+
DefaultExecutorMemoryRequest string
58+
// DefaultExecutorMemoryLimit is the default memory limit for the executor
59+
DefaultExecutorMemoryLimit string
60+
// DefaultExecutorCPURequest is the default CPU request for the executor
61+
DefaultExecutorCPURequest string
62+
// DefaultExecutorCPULimit is the default CPU limit for the executor
63+
DefaultExecutorCPULimit string
5664
// EnableInfraCosts is the flag to enable cost analysis
5765
EnableInfraCosts bool
5866
// ExecutorImage is the image to use for the terraform jobs
@@ -206,6 +214,10 @@ func (r *Render) createTerraformFromTemplate(options Options, stage string) (*ba
206214
terraformv1alpha1.ConfigurationStageLabel: stage,
207215
terraformv1alpha1.ConfigurationUIDLabel: string(r.configuration.GetUID()),
208216
}),
217+
"DefaultExecutorMemoryRequest": options.DefaultExecutorMemoryRequest,
218+
"DefaultExecutorMemoryLimit": options.DefaultExecutorMemoryLimit,
219+
"DefaultExecutorCPURequest": options.DefaultExecutorCPURequest,
220+
"DefaultExecutorCPULimit": options.DefaultExecutorCPULimit,
209221
"Provider": map[string]interface{}{
210222
"Name": r.provider.Name,
211223
"Namespace": r.provider.Namespace,

0 commit comments

Comments
 (0)